From b0c5bc793d768250af95c8f1e623787a4a74d0c7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 9 Mar 2018 09:58:49 +0100 Subject: [PATCH 001/178] Fix "label empty or too long" error #645 --- cdist/config.py | 11 ++++++++++- docs/changelog | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cdist/config.py b/cdist/config.py index 38e225a5..74f68a72 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -333,6 +333,15 @@ class Config(object): family = 0 return family + @staticmethod + def resolve_target_addresses(host, family): + try: + return ipaddr.resolve_target_addresses(host, family) + except: + e = sys.exc_info()[1] + raise cdist.Error(("Error resolving target addresses for host '{}'" + ": {}").format(host, e)) + @classmethod def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, parallel, configuration, remove_remote_files_dirs=False): @@ -353,7 +362,7 @@ class Config(object): family = cls._address_family(args) log.debug("address family: {}".format(family)) - target_host = ipaddr.resolve_target_addresses(host, family) + target_host = cls.resolve_target_addresses(host, family) log.debug("target_host for host \"{}\": {}".format( host, target_host)) diff --git a/docs/changelog b/docs/changelog index 0b5c5f17..3ceae2ec 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * New type: __apt_default_release (Matthijs Kooijman) * Type __file: Add pre-exists state (Matthijs Kooijman) * Type __grafana_dashboard: Add support for stretch + ascii (Nico Schottelius) + * Core: Fix idna (getaddrinfo) unicode tracebak for invalid host name (Darko Poljak) 4.8.0: 2018-02-14 * Core: Skip empty lines in parameter files (Darko Poljak) From 5090038e4ca8d3fbc21fd0cb3ba96c448407d858 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 9 Mar 2018 10:00:14 +0100 Subject: [PATCH 002/178] Add test-remote target - remote is not tested with test target --- Makefile | 3 +++ bin/build-helper | 5 +++++ bin/build-helper.freebsd | 5 +++++ 3 files changed, 13 insertions(+) diff --git a/Makefile b/Makefile index e33be3f2..d727bccc 100644 --- a/Makefile +++ b/Makefile @@ -247,5 +247,8 @@ pub: test: $(helper) $@ +test-remote: + $(helper) $@ + pep8: $(helper) $@ diff --git a/bin/build-helper b/bin/build-helper index 46b139d1..ef15b6c2 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -359,6 +359,11 @@ eof fi ;; + test-remote) + export PYTHONPATH="$(pwd -P)" + python3 -m cdist.test.exec.remote + ;; + pep8) pep8 "${basedir}" "${basedir}/scripts/cdist" | less ;; diff --git a/bin/build-helper.freebsd b/bin/build-helper.freebsd index 183129db..e49b2f7a 100755 --- a/bin/build-helper.freebsd +++ b/bin/build-helper.freebsd @@ -421,6 +421,11 @@ eof fi ;; + test-remote) + export PYTHONPATH="$(pwd -P)" + python3 -m cdist.test.exec.remote + ;; + pep8) pep8 "${basedir}" "${basedir}/scripts/cdist" | less ;; From fc79fe86a243eb6bc842e1582870a3031cdb94d3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 9 Mar 2018 17:25:01 +0100 Subject: [PATCH 003/178] Release 4.8.1 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3ceae2ec..abe36509 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.8.1: 2018-03-09 * Type __consul: Add option for directly downloading on target host (Darko Poljak) * Core: Add -4 and -6 params to force IPv4, IPv6 addresses respectively (Darko Poljak) * Type __package_update_index: Fix messaging (Thomas Eckert) From fe870ba8ba9afa09bd7339b869dedf07ccf57e92 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Mar 2018 11:12:31 +0100 Subject: [PATCH 004/178] Loglevel arguments not parsed but accessed #646 --- cdist/argparse.py | 2 +- docs/changelog | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index fbe2bba6..29620751 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -434,7 +434,7 @@ def get_parsers(): def handle_loglevel(args): - if args.quiet: + if hasattr(args, 'quiet') and args.quiet: args.verbose = _verbosity_level_off logging.root.setLevel(_verbosity_level[args.verbose]) diff --git a/docs/changelog b/docs/changelog index abe36509..a86cc6b0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Core: Fix quiet argument access for bare cdist command (Darko Poljak) + 4.8.1: 2018-03-09 * Type __consul: Add option for directly downloading on target host (Darko Poljak) * Core: Add -4 and -6 params to force IPv4, IPv6 addresses respectively (Darko Poljak) From b58cb810acd3e36c1feb4b27c92415845e32f000 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Mar 2018 23:49:50 +0100 Subject: [PATCH 005/178] Release 4.8.2 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index a86cc6b0..344d18a1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.8.2: 2018-03-10 * Core: Fix quiet argument access for bare cdist command (Darko Poljak) 4.8.1: 2018-03-09 From 8a488591bb190d712fd44588ae727d1eb92fdac3 Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 00:47:33 +0100 Subject: [PATCH 006/178] __prometheus_server: switch to packages; assume prometheus 2.x --- cdist/conf/type/__prometheus_server/man.rst | 24 ++------ cdist/conf/type/__prometheus_server/manifest | 61 ++++++++++++------- .../__prometheus_server/parameter/boolean | 1 + .../__prometheus_server/parameter/required | 2 - 4 files changed, 45 insertions(+), 43 deletions(-) create mode 100644 cdist/conf/type/__prometheus_server/parameter/boolean diff --git a/cdist/conf/type/__prometheus_server/man.rst b/cdist/conf/type/__prometheus_server/man.rst index fadebd3f..33dd3e53 100644 --- a/cdist/conf/type/__prometheus_server/man.rst +++ b/cdist/conf/type/__prometheus_server/man.rst @@ -10,18 +10,12 @@ DESCRIPTION ----------- Install and configure Prometheus (https://prometheus.io/). -This type creates a daemontools-compatible service directory under /service/prometheus. -Daemontools (or something compatible) must be installed (in particular, the command `svc` must be executable). - +Note that due to significant differences between Prometheus 1.x and 2.x, only 2.x is supported. REQUIRED PARAMETERS ------------------- config Prometheus configuration file. It will be saved as /etc/prometheus/prometheus.yml on the target. -listen-address - Passed as web.listen-address. -alertmanager-url - Passed as alertmanager.url OPTIONAL PARAMETERS @@ -32,8 +26,6 @@ rule-files Path to rule files. They will be installed under /etc/prometheus/. You need to include `rule_files: [/etc/prometheus/]` in the config file if you use this. storage-path Where to put data. Default: /data/prometheus. (Directory will be created if needed.) -target-heap-size - Passed as storage.local.target-heap-size. Default: 1/2 of RAM. BOOLEAN PARAMETERS @@ -49,22 +41,16 @@ EXAMPLES PROMPORT=9090 ALERTPORT=9093 - __daemontools - __golang_from_vendor --version 1.8.1 # required for prometheus and many exporters - - require="__daemontools __golang_from_vendor" __prometheus_server \ - --with-daemontools \ + __prometheus_server \ --config "$__manifest/files/prometheus.yml" \ --retention-days 14 \ --storage-path /data/prometheus \ - --listen-address "[::]:$PROMPORT" \ - --rule-files "$__manifest/files/*.rules" \ - --alertmanager-url "http://monitoring1.node.consul:$ALERTPORT,http://monitoring2.node.consul:$ALERTPORT" + --rule-files "$__manifest/files/*.rules" SEE ALSO -------- -:strong:`cdist-type__prometheus_alertmanager`\ (7), :strong:`cdist-type__daemontools`\ (7), +:strong:`cdist-type__prometheus_alertmanager`\ (7), :strong:`cdist-type__grafana_dashboard`\ (7), Prometheus documentation: https://prometheus.io/docs/introduction/overview/ AUTHORS @@ -73,7 +59,7 @@ Kamila Součková COPYING ------- -Copyright \(C) 2017 Kamila Součková. You can redistribute it +Copyright \(C) 2018 Kamila Součková. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index 96717ed6..d19d35b1 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -1,52 +1,69 @@ #!/bin/sh -e -GOBIN=/opt/gocode/bin # where to find go binaries +##### HARD-CODED CONFIG ##################################################### + CONF_DIR=/etc/prometheus CONF=$CONF_DIR/prometheus.yml -LOGLEVEL=info + +##### GET SETTINGS ########################################################## config="$(cat "$__object/parameter/config")" retention_days="$(cat "$__object/parameter/retention-days")" storage_path="$(cat "$__object/parameter/storage-path")" -listen_address="$(cat "$__object/parameter/listen-address")" -alertmanager_url="$(cat "$__object/parameter/alertmanager-url")" -target_heap_size="$(cat "$__object/parameter/target-heap-size")" rule_files="$(cat "$__object/parameter/rule-files")" # explorer in kB => convert; by default we go with 1/2 RAM [ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat $__global/explorer/memory)*1024/2)) +##### INSTALL THE PACKAGE ################################################### -FLAGS="config.file '$CONF' -storage.local.path '$storage_path' -storage.local.target-heap-size $(($target_heap_size)) # in bytes; should be 2/3 of available memory because it may be hungry -storage.local.retention $(($retention_days*24))h # golang doesn't have days :D -web.listen-address '$listen_address' -alertmanager.url '$alertmanager_url' -log.level $LOGLEVEL" +if [ -f "$__object/parameter/install-from-backports" ]; then + os=$(cat "$__global/explorer/os") + lsb_codename=$(cat "$__global/explorer/lsb_codename") -REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')" + case $os in + devuan) + if [ -z "$lsb_codename" ]; then + echo "Command `lsb_release` not functional -- is package `lsb-release` installed?" >&2 + exit 1 + fi + [ "$lsb_codename" = "n/a" ] && lsb_codename='ascii' # TODO this is a devuan bug that should be fixed soon => remove when no longer needed + __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $lsb_codename-backports --component main + require="$require __apt_source/backports" __package_apt prometheus --target-release $lsb_codename-backports + ;; + *) + echo "--install-from-backports is only supported on Devuan -- ignoring" >&2 + echo "send a pull request if you require it" >&2 + ;; + esac +else + __package prometheus +fi -__go_get github.com/prometheus/prometheus/cmd/... +##### PREPARE PATHS AND SUCH ################################################ -__user prometheus --system -require="__user/prometheus" __directory "$storage_path" --owner prometheus --parents -require="__user/prometheus" __directory "$CONF_DIR" --owner prometheus --parents +require="__package_apt/prometheus" __directory "$storage_path" --owner prometheus --parents -__daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $REAL_FLAGS" +##### CONFIGURE ############################################################# -require="$require __directory/$storage_path __user/prometheus" \ +FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $(($retention_days*24))h" + +# TODO it would be neat to restart prometheus on change -- __key_value really should have an --onchange parameter +require="$require __package_apt/prometheus" \ +__key_value prometheus_args --file /etc/default/prometheus --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" + +require="$require __directory/$storage_path __package_apt/prometheus" \ __config_file $CONF \ --source $config \ --group prometheus --mode 640 \ - --onchange "$GOBIN/promtool check-config $CONF && svc -h /service/prometheus" + --onchange "promtool check config $CONF && service prometheus reload" for file in $rule_files; do dest=$CONF_DIR/$(basename $file) - require="$require __directory/$CONF_DIR __user/prometheus" \ + require="$require __package_apt/prometheus" \ __config_file "$dest" \ --source "$file" \ --owner prometheus \ - --onchange "$GOBIN/promtool check-rules '$dest' && svc -h /service/prometheus" + --onchange "promtool check rules '$dest' && service prometheus reload" done diff --git a/cdist/conf/type/__prometheus_server/parameter/boolean b/cdist/conf/type/__prometheus_server/parameter/boolean new file mode 100644 index 00000000..5d15e93d --- /dev/null +++ b/cdist/conf/type/__prometheus_server/parameter/boolean @@ -0,0 +1 @@ +install-from-backports diff --git a/cdist/conf/type/__prometheus_server/parameter/required b/cdist/conf/type/__prometheus_server/parameter/required index 49abf924..04204c7c 100644 --- a/cdist/conf/type/__prometheus_server/parameter/required +++ b/cdist/conf/type/__prometheus_server/parameter/required @@ -1,3 +1 @@ -alertmanager-url config -listen-address From ee6d63325d5e0267be90a70ae0c9db0e35764968 Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 01:28:26 +0100 Subject: [PATCH 007/178] new __prometheus_alertmanager; fixes to __prometheus_server --- .../type/__prometheus_alertmanager/manifest | 62 ++++++++++++++----- .../parameter/boolean | 1 + .../parameter/default/retention-days | 1 + .../parameter/optional | 1 + .../parameter/required | 1 - cdist/conf/type/__prometheus_server/manifest | 13 ++-- .../parameter/default/target-heap-size | 1 - .../__prometheus_server/parameter/optional | 1 - 8 files changed, 57 insertions(+), 24 deletions(-) create mode 100644 cdist/conf/type/__prometheus_alertmanager/parameter/boolean create mode 100644 cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days delete mode 100644 cdist/conf/type/__prometheus_server/parameter/default/target-heap-size diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index 0dbce3c2..926ad712 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -1,34 +1,64 @@ #!/bin/sh -e -GOBIN=/opt/gocode/bin # where to find go binaries +##### HARD-CODED CONFIG ##################################################### + CONF_DIR=/etc/prometheus -LOGLEVEL=info CONF=$CONF_DIR/alertmanager.yml -### Prometheus server ####################################################### +##### GET SETTINGS ########################################################## config="$(cat "$__object/parameter/config")" +retention_days="$(cat "$__object/parameter/retention-days")" storage_path="$(cat "$__object/parameter/storage-path")" -listen_address="$(cat "$__object/parameter/listen-address")" +# listen_address="$(cat "$__object/parameter/listen-address")" -FLAGS="config.file '$CONF' -storage.path '$storage_path' -web.listen-address '$listen_address' -log.level $LOGLEVEL" +##### INSTALL THE PACKAGE ################################################### -REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')" +require_pkg="" # what to require if I want to require "the package" +if [ -f "$__object/parameter/install-from-backports" ]; then + os=$(cat "$__global/explorer/os") + lsb_codename=$(cat "$__global/explorer/lsb_codename") -__go_get github.com/prometheus/alertmanager/cmd/... + case $os in + devuan) + if [ -z "$lsb_codename" ]; then + echo "Command `lsb_release` not functional -- is package `lsb-release` installed?" >&2 + exit 1 + fi + [ "$lsb_codename" = "n/a" ] && lsb_codename='ascii' # TODO this is a devuan bug that should be fixed soon => remove when no longer needed + __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $lsb_codename-backports --component main + require="$require __apt_source/backports" __package_apt prometheus-alertmanager --target-release $lsb_codename-backports + require_pkg="__package_apt/prometheus-alertmanager" + ;; + *) + echo "--install-from-backports is only supported on Devuan -- ignoring" >&2 + echo "send a pull request if you require it" >&2 + ;; + esac +else + __package prometheus-alertmanager + require_pkg="__package/prometheus-alertmanager" +fi -__user prometheus --system -require="__user/prometheus" __directory "$storage_path" --owner prometheus --parents -require="__user/prometheus" __directory "$CONF_DIR" --owner prometheus --parents +##### PREPARE PATHS AND SUCH ################################################ -__daemontools_service alertmanager --run "setuidgid prometheus $GOBIN/alertmanager $REAL_FLAGS" +require="$require $require_pkg" __directory "$storage_path" --owner prometheus --parents -require="$require __directory/$storage_path __user/prometheus" \ +# TODO this is a bug in the init script, patching it like this is awful and it should be reported +require="$require $require_pkg" \ +__key_value alertmanager_fix_init_script --file /etc/init.d/prometheus-alertmanager --key="NAME" --value="prometheus-alertmanager" --delimiter "=" + + +##### CONFIGURE ############################################################# + +FLAGS="--storage.path $storage_path --data.retention $(($retention_days*24))h --web.listen-address [::]:9093" + +require="$require $require_pkg" \ +__key_value alertmanager_args --file /etc/default/prometheus-alertmanager --key="ARGS" --value="\"$FLAGS\"" --delimiter "=" + +require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ --source $config \ --group prometheus --mode 640 \ - --onchange "svc -h /service/alertmanager" # TODO when a config-check tool is available, check config here + --onchange "service prometheus-alertmanager reload" # TODO when a config-check tool is available, check config here diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/boolean b/cdist/conf/type/__prometheus_alertmanager/parameter/boolean new file mode 100644 index 00000000..5d15e93d --- /dev/null +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/boolean @@ -0,0 +1 @@ +install-from-backports diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days b/cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days new file mode 100644 index 00000000..d61f00d8 --- /dev/null +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days @@ -0,0 +1 @@ +90 diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/optional b/cdist/conf/type/__prometheus_alertmanager/parameter/optional index f99d0d37..7fe79009 100644 --- a/cdist/conf/type/__prometheus_alertmanager/parameter/optional +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/optional @@ -1 +1,2 @@ storage-path +retention-days diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/required b/cdist/conf/type/__prometheus_alertmanager/parameter/required index 02cb49d0..04204c7c 100644 --- a/cdist/conf/type/__prometheus_alertmanager/parameter/required +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/required @@ -1,2 +1 @@ config -listen-address diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index d19d35b1..a5c31b4e 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -17,6 +17,7 @@ rule_files="$(cat "$__object/parameter/rule-files")" ##### INSTALL THE PACKAGE ################################################### +require_pkg="" # what to require if I want to require "the package" if [ -f "$__object/parameter/install-from-backports" ]; then os=$(cat "$__global/explorer/os") lsb_codename=$(cat "$__global/explorer/lsb_codename") @@ -30,6 +31,7 @@ if [ -f "$__object/parameter/install-from-backports" ]; then [ "$lsb_codename" = "n/a" ] && lsb_codename='ascii' # TODO this is a devuan bug that should be fixed soon => remove when no longer needed __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $lsb_codename-backports --component main require="$require __apt_source/backports" __package_apt prometheus --target-release $lsb_codename-backports + require_pkg="__package_apt/prometheus" ;; *) echo "--install-from-backports is only supported on Devuan -- ignoring" >&2 @@ -38,21 +40,22 @@ if [ -f "$__object/parameter/install-from-backports" ]; then esac else __package prometheus + require_pkg="__package/prometheus" fi ##### PREPARE PATHS AND SUCH ################################################ -require="__package_apt/prometheus" __directory "$storage_path" --owner prometheus --parents +require="$require $require_pkg" __directory "$storage_path" --owner prometheus --parents ##### CONFIGURE ############################################################# -FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $(($retention_days*24))h" +FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $(($retention_days*24))h --web.listen-address [::]:9090" # TODO it would be neat to restart prometheus on change -- __key_value really should have an --onchange parameter -require="$require __package_apt/prometheus" \ +require="$require $require_pkg" \ __key_value prometheus_args --file /etc/default/prometheus --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" -require="$require __directory/$storage_path __package_apt/prometheus" \ +require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ --source $config \ --group prometheus --mode 640 \ @@ -60,7 +63,7 @@ __config_file $CONF \ for file in $rule_files; do dest=$CONF_DIR/$(basename $file) - require="$require __package_apt/prometheus" \ + require="$require $require_pkg" \ __config_file "$dest" \ --source "$file" \ --owner prometheus \ diff --git a/cdist/conf/type/__prometheus_server/parameter/default/target-heap-size b/cdist/conf/type/__prometheus_server/parameter/default/target-heap-size deleted file mode 100644 index 865faf10..00000000 --- a/cdist/conf/type/__prometheus_server/parameter/default/target-heap-size +++ /dev/null @@ -1 +0,0 @@ -auto diff --git a/cdist/conf/type/__prometheus_server/parameter/optional b/cdist/conf/type/__prometheus_server/parameter/optional index 4d8d8f3e..cb437211 100644 --- a/cdist/conf/type/__prometheus_server/parameter/optional +++ b/cdist/conf/type/__prometheus_server/parameter/optional @@ -1,4 +1,3 @@ -target-heap-size retention-days rule-files storage-path From 34aefbc0f6ea3340c6661f1c23be3ae1fe5f0a71 Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 01:35:16 +0100 Subject: [PATCH 008/178] __prometheus_{server,alertmanager}: new man pages --- .../type/__prometheus_alertmanager/man.rst | 25 ++++++++----------- .../type/__prometheus_alertmanager/manifest | 1 - cdist/conf/type/__prometheus_server/man.rst | 4 ++- 3 files changed, 13 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__prometheus_alertmanager/man.rst b/cdist/conf/type/__prometheus_alertmanager/man.rst index ba99e7c8..894100c8 100644 --- a/cdist/conf/type/__prometheus_alertmanager/man.rst +++ b/cdist/conf/type/__prometheus_alertmanager/man.rst @@ -18,19 +18,20 @@ REQUIRED PARAMETERS ------------------- config Alertmanager configuration file. It will be saved as /etc/alertmanager/alertmanager.yml on the target. -listen-address - Passed as web.listen-address. OPTIONAL PARAMETERS ------------------- storage-path Where to put data. Default: /data/alertmanager. (Directory will be created if needed.) +retention-days + How long to retain data. Default: 90 days. BOOLEAN PARAMETERS ------------------ -None +install-from-backports + Valid on Devuan only. Will enable the backports apt source and install the package from there. Useful for getting a newer version. EXAMPLES @@ -38,21 +39,15 @@ EXAMPLES .. code-block:: sh - ALERTPORT=9093 - - __daemontools - __golang_from_vendor --version 1.8.1 # required for prometheus and many exporters - - require="__daemontools __golang_from_vendor" __prometheus_alertmanager \ - --with-daemontools \ - --config "$__manifest/files/alertmanager.yml" \ - --storage-path /data/alertmanager \ - --listen-address "[::]:$ALERTPORT" + __prometheus_alertmanager \ + --install-from-backports \ + --config "$__manifest/files/alertmanager.yml" \ + --storage-path /data/alertmanager SEE ALSO -------- -:strong:`cdist-type__prometheus_server`\ (7), :strong:`cdist-type__daemontools`\ (7), +:strong:`cdist-type__prometheus_server`\ (7), :strong:`cdist-type__grafana_dashboard`\ (7), Prometheus alerting documentation: https://prometheus.io/docs/alerting/overview/ AUTHORS @@ -61,7 +56,7 @@ Kamila Součková COPYING ------- -Copyright \(C) 2017 Kamila Součková. You can redistribute it +Copyright \(C) 2018 Kamila Součková. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index 926ad712..8278170a 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -48,7 +48,6 @@ require="$require $require_pkg" __directory "$storage_path" --owner prometheus - require="$require $require_pkg" \ __key_value alertmanager_fix_init_script --file /etc/init.d/prometheus-alertmanager --key="NAME" --value="prometheus-alertmanager" --delimiter "=" - ##### CONFIGURE ############################################################# FLAGS="--storage.path $storage_path --data.retention $(($retention_days*24))h --web.listen-address [::]:9093" diff --git a/cdist/conf/type/__prometheus_server/man.rst b/cdist/conf/type/__prometheus_server/man.rst index 33dd3e53..4308e6c4 100644 --- a/cdist/conf/type/__prometheus_server/man.rst +++ b/cdist/conf/type/__prometheus_server/man.rst @@ -30,7 +30,8 @@ storage-path BOOLEAN PARAMETERS ------------------ -None +install-from-backports + Valid on Devuan only. Will enable the backports apt source and install the package from there. Useful for getting a newer version. EXAMPLES @@ -42,6 +43,7 @@ EXAMPLES ALERTPORT=9093 __prometheus_server \ + --install-from-backports \ --config "$__manifest/files/prometheus.yml" \ --retention-days 14 \ --storage-path /data/prometheus \ From 7ae24cf67785d98c8877e363cf53ad411f2d601b Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 10:52:25 +0100 Subject: [PATCH 009/178] __grafana_dashboard: cleanup --- cdist/conf/type/__grafana_dashboard/manifest | 54 ++++++++------------ 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 8f089367..e62bd15f 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -7,44 +7,34 @@ case $os in debian|devuan) case $os_version in 8*|jessie) - __apt_key_uri grafana \ - --name 'Grafana Release Signing Key' \ - --uri https://packagecloud.io/gpg.key - - require="__apt_key_uri/grafana" __apt_source grafana \ - --uri https://packagecloud.io/grafana/stable/debian/ \ - --distribution jessie \ - --component main - - __package apt-transport-https - - require="__apt_source/grafana __package/apt-transport-https" __package grafana - require="__package/grafana" __start_on_boot grafana-server + apt_source_distribution=jessie ;; 9*|ascii/ceres) - __apt_key_uri grafana \ - --name 'Grafana Release Signing Key' \ - --uri https://packagecloud.io/gpg.key - - require="__apt_key_uri/grafana" __apt_source grafana \ - --uri https://packagecloud.io/grafana/stable/debian/ \ - --distribution stretch \ - --component main - - __package apt-transport-https - - require="__apt_source/grafana __package/apt-transport-https" __package grafana - require="__package/grafana" __start_on_boot grafana-server + apt_source_distribution=stretch ;; - *) - echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" + echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" >&2 exit 1 - ;; + ;; esac - ;; + + __apt_key_uri grafana \ + --name 'Grafana Release Signing Key' \ + --uri https://packagecloud.io/gpg.key + + require="$require __apt_key_uri/grafana" __apt_source grafana \ + --uri https://packagecloud.io/grafana/stable/debian/ \ + --distribution $apt_source_distribution \ + --component main + + __package apt-transport-https + + require="$require __apt_source/grafana __package/apt-transport-https" __package grafana + require="$require __package/grafana" __start_on_boot grafana-server + require="$require __start_on_boot/grafana-server" __process grafana-server --start "service grafana-server start" + ;; *) - echo "Don't know how to install Grafana on $os. Send us a pull request!" + echo "Don't know how to install Grafana on $os. Send us a pull request!" >&2 exit 1 - ;; + ;; esac From 787f5237e5f3c274c94d7b647f874df14c9ccc71 Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 10:55:21 +0100 Subject: [PATCH 010/178] __prometheus_{server,alertmanager}: use os_version instead of lsb_release works without the `lsb-release` package and actually seems to be more reliable, even though it is inconsistent between debian and devuan --- .../conf/type/__prometheus_alertmanager/manifest | 16 ++++++---------- cdist/conf/type/__prometheus_server/manifest | 16 ++++++---------- 2 files changed, 12 insertions(+), 20 deletions(-) diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index 8278170a..87546f47 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -17,22 +17,18 @@ storage_path="$(cat "$__object/parameter/storage-path")" require_pkg="" # what to require if I want to require "the package" if [ -f "$__object/parameter/install-from-backports" ]; then os=$(cat "$__global/explorer/os") - lsb_codename=$(cat "$__global/explorer/lsb_codename") + os_version=$(cat "$__global/explorer/os_version") case $os in devuan) - if [ -z "$lsb_codename" ]; then - echo "Command `lsb_release` not functional -- is package `lsb-release` installed?" >&2 - exit 1 - fi - [ "$lsb_codename" = "n/a" ] && lsb_codename='ascii' # TODO this is a devuan bug that should be fixed soon => remove when no longer needed - __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $lsb_codename-backports --component main - require="$require __apt_source/backports" __package_apt prometheus-alertmanager --target-release $lsb_codename-backports + [ "$os_version" = "ascii/ceres" ] && os_version='ascii' # "ascii" used in the repo URLs + __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $os_version-backports --component main + require="$require __apt_source/backports" __package_apt prometheus-alertmanager --target-release $os_version-backports require_pkg="__package_apt/prometheus-alertmanager" ;; *) - echo "--install-from-backports is only supported on Devuan -- ignoring" >&2 - echo "send a pull request if you require it" >&2 + echo "--install-from-backports is only supported on Devuan -- ignoring." >&2 + echo "Send a pull request if you require it." >&2 ;; esac else diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index a5c31b4e..a7ba7884 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -20,22 +20,18 @@ rule_files="$(cat "$__object/parameter/rule-files")" require_pkg="" # what to require if I want to require "the package" if [ -f "$__object/parameter/install-from-backports" ]; then os=$(cat "$__global/explorer/os") - lsb_codename=$(cat "$__global/explorer/lsb_codename") + os_version=$(cat "$__global/explorer/os_version") case $os in devuan) - if [ -z "$lsb_codename" ]; then - echo "Command `lsb_release` not functional -- is package `lsb-release` installed?" >&2 - exit 1 - fi - [ "$lsb_codename" = "n/a" ] && lsb_codename='ascii' # TODO this is a devuan bug that should be fixed soon => remove when no longer needed - __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $lsb_codename-backports --component main - require="$require __apt_source/backports" __package_apt prometheus --target-release $lsb_codename-backports + [ "$os_version" = "ascii/ceres" ] && os_version='ascii' # "ascii" used in the repo URLs + __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $os_version-backports --component main + require="$require __apt_source/backports" __package_apt prometheus --target-release $os_version-backports require_pkg="__package_apt/prometheus" ;; *) - echo "--install-from-backports is only supported on Devuan -- ignoring" >&2 - echo "send a pull request if you require it" >&2 + echo "--install-from-backports is only supported on Devuan -- ignoring." >&2 + echo "Send a pull request if you require it." >&2 ;; esac else From e63c3429da906510b16587888f98490be87bc114 Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 11:07:59 +0100 Subject: [PATCH 011/178] __prometheus_{server,alertmanager}: update man pages --- cdist/conf/type/__prometheus_alertmanager/man.rst | 3 +-- cdist/conf/type/__prometheus_server/man.rst | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__prometheus_alertmanager/man.rst b/cdist/conf/type/__prometheus_alertmanager/man.rst index 894100c8..67e97eaf 100644 --- a/cdist/conf/type/__prometheus_alertmanager/man.rst +++ b/cdist/conf/type/__prometheus_alertmanager/man.rst @@ -10,8 +10,7 @@ DESCRIPTION ----------- Install and configure Prometheus Alertmanager (https://prometheus.io/docs/alerting/alertmanager/). -This type create a daemontools-compatible service directory under /service/prometheus. -Daemontools (or something compatible) must be installed (in particular, the command `svc` must be executable). +Note that due to significant differences between Prometheus 1.x and 2.x, only 2.x is supported. It is your responsibility to make sure that your package manager installs 2.x. (On Devuan Ascii, the parameter `--install-from-backports` helps.) REQUIRED PARAMETERS diff --git a/cdist/conf/type/__prometheus_server/man.rst b/cdist/conf/type/__prometheus_server/man.rst index 4308e6c4..ab6a3c9b 100644 --- a/cdist/conf/type/__prometheus_server/man.rst +++ b/cdist/conf/type/__prometheus_server/man.rst @@ -10,7 +10,7 @@ DESCRIPTION ----------- Install and configure Prometheus (https://prometheus.io/). -Note that due to significant differences between Prometheus 1.x and 2.x, only 2.x is supported. +Note that due to significant differences between Prometheus 1.x and 2.x, only 2.x is supported. It is your responsibility to make sure that your package manager installs 2.x. (On Devuan Ascii, the parameter `--install-from-backports` helps.) REQUIRED PARAMETERS ------------------- From f9443532a19ce9fb179970498014d14e0fd92ed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Sun, 11 Mar 2018 18:01:38 +0100 Subject: [PATCH 012/178] __key_value: --onchange parameter added (#648) --- cdist/conf/type/__key_value/gencode-remote | 15 ++++++++++++--- cdist/conf/type/__key_value/man.rst | 2 ++ .../type/__key_value/parameter/default/onchange | 0 cdist/conf/type/__key_value/parameter/optional | 1 + 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 cdist/conf/type/__key_value/parameter/default/onchange diff --git a/cdist/conf/type/__key_value/gencode-remote b/cdist/conf/type/__key_value/gencode-remote index 7a60f94b..13cc27c7 100755 --- a/cdist/conf/type/__key_value/gencode-remote +++ b/cdist/conf/type/__key_value/gencode-remote @@ -23,13 +23,14 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" +fire_onchange='' if [ "$state_is" = "$state_should" ]; then exit 0 fi # here we check only if the states are valid, -# emmit messages and +# emit messages and # let awk do the work ... case "$state_should" in absent) @@ -39,6 +40,7 @@ case "$state_should" in ;; wrongformat|wrongvalue|present) echo "remove" >> "$__messages_out" + fire_onchange=1 ;; *) echo "Unknown explorer state: $state_is" >&2 @@ -50,12 +52,15 @@ case "$state_should" in case "$state_is" in nosuchfile) echo "create" >> "$__messages_out" + fire_onchange=1 ;; absent) echo "insert" >> "$__messages_out" + fire_onchange=1 ;; wrongformated|wrongvalue) echo "change" >> "$__messages_out" + fire_onchange=1 ;; present) # nothing to do @@ -67,9 +72,13 @@ case "$state_should" in esac ;; *) - echo "Unknown state: $state_should" >&2 - exit 1 + echo "Unknown state: $state_should" >&2 + exit 1 ;; esac cat "$__type/files/remote_script.sh" + +if [ -n "$fire_onchange" ]; then + cat "$__object/parameter/onchange" +fi diff --git a/cdist/conf/type/__key_value/man.rst b/cdist/conf/type/__key_value/man.rst index f069d989..34e4aab2 100644 --- a/cdist/conf/type/__key_value/man.rst +++ b/cdist/conf/type/__key_value/man.rst @@ -34,6 +34,8 @@ comment but only if the key or value must be changed. You need to ensure yourself that the line is prefixed with the correct comment sign. (for example # or ; or wathever ..) +onchange + The code to run if the key or value changes (i.e. is inserted, removed or replaced). BOOLEAN PARAMETERS diff --git a/cdist/conf/type/__key_value/parameter/default/onchange b/cdist/conf/type/__key_value/parameter/default/onchange new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__key_value/parameter/optional b/cdist/conf/type/__key_value/parameter/optional index 666be2ae..d4b8cac0 100644 --- a/cdist/conf/type/__key_value/parameter/optional +++ b/cdist/conf/type/__key_value/parameter/optional @@ -2,3 +2,4 @@ key value state comment +onchange From ca42fe6ee3471907daf1eab08ec519d312481110 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 11 Mar 2018 18:01:03 +0100 Subject: [PATCH 013/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 344d18a1..834488e8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * __key_value: Add onchange parameter (Kamila Součková) + 4.8.2: 2018-03-10 * Core: Fix quiet argument access for bare cdist command (Darko Poljak) From f03a0e586e3d867d65244b586fee6516c8854a6c Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Sun, 11 Mar 2018 11:40:47 +0100 Subject: [PATCH 014/178] __prometheus_{server,alertmanager}: restart when cli parameters change --- cdist/conf/type/__prometheus_alertmanager/manifest | 8 ++++++-- cdist/conf/type/__prometheus_server/manifest | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index 87546f47..be50b71e 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -42,14 +42,18 @@ require="$require $require_pkg" __directory "$storage_path" --owner prometheus - # TODO this is a bug in the init script, patching it like this is awful and it should be reported require="$require $require_pkg" \ -__key_value alertmanager_fix_init_script --file /etc/init.d/prometheus-alertmanager --key="NAME" --value="prometheus-alertmanager" --delimiter "=" +__key_value alertmanager_fix_init_script --file /etc/init.d/prometheus-alertmanager \ + --key "NAME" --value "prometheus-alertmanager" --delimiter "=" \ + --onchange "service prometheus-alertmanager restart" ##### CONFIGURE ############################################################# FLAGS="--storage.path $storage_path --data.retention $(($retention_days*24))h --web.listen-address [::]:9093" require="$require $require_pkg" \ -__key_value alertmanager_args --file /etc/default/prometheus-alertmanager --key="ARGS" --value="\"$FLAGS\"" --delimiter "=" +__key_value alertmanager_args --file /etc/default/prometheus-alertmanager \ + --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" \ + --onchange "service prometheus-alertmanager restart" require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index a7ba7884..84ba53cf 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -49,7 +49,9 @@ FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $(($retention_ # TODO it would be neat to restart prometheus on change -- __key_value really should have an --onchange parameter require="$require $require_pkg" \ -__key_value prometheus_args --file /etc/default/prometheus --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" +__key_value prometheus_args --file /etc/default/prometheus \ + --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" \ + --onchange "service prometheus restart" require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ From e4f263ecb72415a2f7759235eb87782359ac1c16 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 11 Mar 2018 21:11:26 +0100 Subject: [PATCH 015/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 834488e8..88ff926e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * __key_value: Add onchange parameter (Kamila Součková) + * __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: + Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) 4.8.2: 2018-03-10 * Core: Fix quiet argument access for bare cdist command (Darko Poljak) From f8812288783a974e4f7ea61257e85eab2c4d0957 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Mar 2018 15:09:13 +0100 Subject: [PATCH 016/178] Fix manpage generation. --- cdist/sphinxext/manpage.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py index 97b41f03..a436421d 100644 --- a/cdist/sphinxext/manpage.py +++ b/cdist/sphinxext/manpage.py @@ -27,13 +27,12 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): def __init__(self, builder): super().__init__(builder) - self.translator_class = ( - self.builder.translator_class or ManualPageTranslator) class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): name = 'cman' + default_translator_class = ManualPageTranslator def write(self, *ignored): docwriter = ManualPageWriter(self) From f2fbb92fdd73c1b8cf78ba878850697065732e8b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Mar 2018 15:17:14 +0100 Subject: [PATCH 017/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 88ff926e..5c98ba88 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * __key_value: Add onchange parameter (Kamila Součková) * __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) + * Documentation: Fix manpage generation (Darko Poljak) 4.8.2: 2018-03-10 * Core: Fix quiet argument access for bare cdist command (Darko Poljak) From 8b86c6fc95806562bde77af7212bb1d88e404a1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Mon, 12 Mar 2018 12:30:22 +0100 Subject: [PATCH 018/178] Add new type __docker_swarm --- .../type/__docker_swarm/explorer/swarm-state | 21 ++++++++ cdist/conf/type/__docker_swarm/gencode-remote | 46 +++++++++++++++++ cdist/conf/type/__docker_swarm/man.rst | 49 +++++++++++++++++++ .../__docker_swarm/parameter/default/state | 1 + .../type/__docker_swarm/parameter/optional | 1 + cdist/conf/type/__docker_swarm/singleton | 0 6 files changed, 118 insertions(+) create mode 100755 cdist/conf/type/__docker_swarm/explorer/swarm-state create mode 100755 cdist/conf/type/__docker_swarm/gencode-remote create mode 100644 cdist/conf/type/__docker_swarm/man.rst create mode 100644 cdist/conf/type/__docker_swarm/parameter/default/state create mode 100644 cdist/conf/type/__docker_swarm/parameter/optional create mode 100644 cdist/conf/type/__docker_swarm/singleton diff --git a/cdist/conf/type/__docker_swarm/explorer/swarm-state b/cdist/conf/type/__docker_swarm/explorer/swarm-state new file mode 100755 index 00000000..9c1bc32d --- /dev/null +++ b/cdist/conf/type/__docker_swarm/explorer/swarm-state @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +docker info 2>/dev/null | grep "^Swarm: " | cut -d " " -f 2- diff --git a/cdist/conf/type/__docker_swarm/gencode-remote b/cdist/conf/type/__docker_swarm/gencode-remote new file mode 100755 index 00000000..4b199a02 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +state=$(cat "${__object:?}/parameter/state") +swarm_state="$(cat "${__object}/explorer/swarm-state")" + +if [ -z "${swarm_state}" ]; then + echo "Unable to determine Swarm state. Is compatible version of Docker installed?" >&2 + + exit 1 +fi + +case "${state}" in + absent) + if [ "${swarm_state}" = "active" ]; then + echo "docker swarm leave --force" + fi + ;; + present) + if [ "${swarm_state}" = "inactive" ]; then + echo "docker swarm init" + fi + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_swarm/man.rst b/cdist/conf/type/__docker_swarm/man.rst new file mode 100644 index 00000000..7b0a57d3 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/man.rst @@ -0,0 +1,49 @@ +cdist-type__docker_swarm(7) +=========================== + +NAME +---- + +cdist-type__docker_swarm - Manage Swarm + +DESCRIPTION +----------- + +This type can initialize Docker swarm mode. For more information about swarm +mode, see `Swarm mode overview `_. + +OPTIONAL PARAMETERS +------------------- + +state + 'present' or 'absent', defaults to 'present' where: + + present + Swarm is initialized + absent + Swarm is left + +EXAMPLES +-------- + +.. code-block:: sh + + # Initalizes a swarm + __docker_swarm + + # Leaves a swarm + __docker_swarm --state absent + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_swarm/parameter/default/state b/cdist/conf/type/__docker_swarm/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_swarm/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_swarm/parameter/optional b/cdist/conf/type/__docker_swarm/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/parameter/optional @@ -0,0 +1 @@ +state diff --git a/cdist/conf/type/__docker_swarm/singleton b/cdist/conf/type/__docker_swarm/singleton new file mode 100644 index 00000000..e69de29b From 7184228536d7c879730ebea2c984ed50b173f835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Tue, 13 Mar 2018 11:26:25 +0100 Subject: [PATCH 019/178] Add new type __docker_stack --- .../type/__docker_stack/explorer/stack-exists | 25 ++++++++ cdist/conf/type/__docker_stack/gencode-remote | 63 +++++++++++++++++++ cdist/conf/type/__docker_stack/man.rst | 54 ++++++++++++++++ .../parameter/default/compose-file | 0 .../__docker_stack/parameter/default/state | 1 + .../type/__docker_stack/parameter/optional | 2 + 6 files changed, 145 insertions(+) create mode 100755 cdist/conf/type/__docker_stack/explorer/stack-exists create mode 100755 cdist/conf/type/__docker_stack/gencode-remote create mode 100644 cdist/conf/type/__docker_stack/man.rst create mode 100644 cdist/conf/type/__docker_stack/parameter/default/compose-file create mode 100644 cdist/conf/type/__docker_stack/parameter/default/state create mode 100644 cdist/conf/type/__docker_stack/parameter/optional diff --git a/cdist/conf/type/__docker_stack/explorer/stack-exists b/cdist/conf/type/__docker_stack/explorer/stack-exists new file mode 100755 index 00000000..4f511821 --- /dev/null +++ b/cdist/conf/type/__docker_stack/explorer/stack-exists @@ -0,0 +1,25 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if docker stack ls | grep -q "^${__object_id:?} "; then + echo 1 +else + echo 0 +fi diff --git a/cdist/conf/type/__docker_stack/gencode-remote b/cdist/conf/type/__docker_stack/gencode-remote new file mode 100755 index 00000000..0d47dc52 --- /dev/null +++ b/cdist/conf/type/__docker_stack/gencode-remote @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +stack="${__object_id:?}" +state=$(cat "${__object:?}/parameter/state") + +case "${state}" in + absent) + stack_exists=$(cat "${__object:?}/explorer/stack-exists") + + if [ "${stack_exists}" -ne 1 ]; then + exit 0 + fi + + echo "docker stack rm ${stack}" + ;; + present) + compose_file=$(cat "${__object}/parameter/compose-file") + + if [ -z "${compose_file}" ]; then + exit 0 + fi + + if [ "${compose_file}" = "-" ]; then + compose_file="${__object}/stdin" + fi + + cat <<-EOF + compose_file="\$(mktemp cdist.XXXXXXXXXX)" + + base64 -d > "\${compose_file}" << eof + $(base64 "${compose_file}") + eof + + docker stack deploy --compose-file "\${compose_file}" \ + --prune ${stack} + + rm "\${compose_file}" + EOF + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_stack/man.rst b/cdist/conf/type/__docker_stack/man.rst new file mode 100644 index 00000000..d0597c25 --- /dev/null +++ b/cdist/conf/type/__docker_stack/man.rst @@ -0,0 +1,54 @@ +cdist-type__docker_stack(7) +=========================== + +NAME +---- + +cdist-type__docker_stack - Manage Docker stacks + +DESCRIPTION +----------- + +This type manages service stacks. + +.. note:: + Since there is no easy way to tell whether a stack needs to be updated, + `docker stack deploy` is being run every time this type is invoked. + However, it does not mean this type is not idempotent. If Docker does not + detect changes, the existing stack will not be updated. + +OPTIONAL PARAMETERS +------------------- + +compose-file + Path to the compose file. If it is '-' (dash), read standard input. + +state + 'present' or 'absent', defaults to 'present' where: + + present + the stack is deployed + absent + the stack is removed + +EXAMPLES +-------- + +.. code-block:: sh + + # Deploys 'foo' stack defined in 'docker-compose.yml' compose file + __docker_stack foo --compose-file docker-compose.yml + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_stack/parameter/default/compose-file b/cdist/conf/type/__docker_stack/parameter/default/compose-file new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__docker_stack/parameter/default/state b/cdist/conf/type/__docker_stack/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_stack/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_stack/parameter/optional b/cdist/conf/type/__docker_stack/parameter/optional new file mode 100644 index 00000000..b3457bd3 --- /dev/null +++ b/cdist/conf/type/__docker_stack/parameter/optional @@ -0,0 +1,2 @@ +compose-file +state From 17fa27a56ae8cc72c0ccdbdd5cc30f222fa0612a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Mar 2018 17:31:50 +0100 Subject: [PATCH 020/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5c98ba88..8fd5ae81 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) * Documentation: Fix manpage generation (Darko Poljak) + * New type: __docker_swarm (Ľubomír Kučera) 4.8.2: 2018-03-10 * Core: Fix quiet argument access for bare cdist command (Darko Poljak) From bba4c0ef67d2acd09f0a00e5b0ca23a27b897b5d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Mar 2018 18:49:40 +0100 Subject: [PATCH 021/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8fd5ae81..399287c6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) * Documentation: Fix manpage generation (Darko Poljak) * New type: __docker_swarm (Ľubomír Kučera) + * New type: __docker_stack (Ľubomír Kučera) 4.8.2: 2018-03-10 * Core: Fix quiet argument access for bare cdist command (Darko Poljak) From 26a8827ec204c64864784a8d58d61cd6e6f63fb8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 15 Mar 2018 06:33:05 +0100 Subject: [PATCH 022/178] Add gitattributes: export-ignore entries. --- .gitattributes | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..76eac516 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +.gitignore export-ignore +.gitattributes export-ignore +.gitkeep export-ignore +docs/speeches export-ignore +docs/video export-ignore From d4a7b298a6eb62abb33c001f124ca9477f0afd84 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 15 Mar 2018 13:49:22 +0100 Subject: [PATCH 023/178] Return translator_class for older sphinx versions. --- cdist/sphinxext/manpage.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py index a436421d..135fe22e 100644 --- a/cdist/sphinxext/manpage.py +++ b/cdist/sphinxext/manpage.py @@ -27,6 +27,8 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): def __init__(self, builder): super().__init__(builder) + self.translator_class = ( + self.builder.translator_class or ManualPageTranslator) class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): From a06ee6f9f190e9ce3875da682e49275e3a9a2a04 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 15 Mar 2018 14:00:25 +0100 Subject: [PATCH 024/178] Fix changelog. --- docs/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog b/docs/changelog index 399287c6..64ab1157 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,8 +2,8 @@ Changelog --------- next: - * __key_value: Add onchange parameter (Kamila Součková) - * __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: + * Type __key_value: Add onchange parameter (Kamila Součková) + * Types __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) * Documentation: Fix manpage generation (Darko Poljak) * New type: __docker_swarm (Ľubomír Kučera) From ef134496877bf6854b909a01bdcae652b702428f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 15 Mar 2018 18:01:23 +0100 Subject: [PATCH 025/178] Fix sphinx warnings. --- cdist/conf/type/__consul/man.rst | 2 + cdist/conf/type/__install_stage/man.rst | 6 +- docs/src/cdist-saving-output-streams.rst | 2 +- docs/src/conf.py | 5 - docs/src/docutils.conf | 2 + docs/src/man1/cdist.rst | 186 ++++++++--------------- 6 files changed, 70 insertions(+), 133 deletions(-) create mode 100644 docs/src/docutils.conf diff --git a/cdist/conf/type/__consul/man.rst b/cdist/conf/type/__consul/man.rst index 401f0c26..5b2db50a 100644 --- a/cdist/conf/type/__consul/man.rst +++ b/cdist/conf/type/__consul/man.rst @@ -40,9 +40,11 @@ MESSAGES If consul binary is created using __staged_file then underlaying __file type messages are emitted. If consul binary is created by direct method then the following messages are emitted: + /usr/local/bin/consul created consul binary was created + EXAMPLES -------- diff --git a/cdist/conf/type/__install_stage/man.rst b/cdist/conf/type/__install_stage/man.rst index e33e1e90..fd764693 100644 --- a/cdist/conf/type/__install_stage/man.rst +++ b/cdist/conf/type/__install_stage/man.rst @@ -17,9 +17,9 @@ REQUIRED PARAMETERS uri The uri from which to fetch the tarball. Can be anything understood by curl, e.g: - | http://path/to/stage.tgz - | tftp:///path/to/stage.tgz - | file:///local/path/stage.tgz + | http://path/to/stage.tgz + | tftp:///path/to/stage.tgz + | file:///local/path/stage.tgz OPTIONAL PARAMETERS diff --git a/docs/src/cdist-saving-output-streams.rst b/docs/src/cdist-saving-output-streams.rst index 28067cac..da66f754 100644 --- a/docs/src/cdist-saving-output-streams.rst +++ b/docs/src/cdist-saving-output-streams.rst @@ -9,7 +9,7 @@ during a config run, hidden in all other output. Now all created output is bound to the context where it was produced. Saving output streams include stdout and stderr of init manifest, remote -commands and for each object stdout and stderr of manifest, gencode-* and code-*. +commands and for each object stdout and stderr of manifest, gencode-\* and code-\*. Output stream files are created only if some output is produced. For more info on these cache files see `Local cache overview `_. diff --git a/docs/src/conf.py b/docs/src/conf.py index a63a14ff..8ed48324 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -161,11 +161,6 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The empty string is equivalent to '%b %d, %Y'. # html_last_updated_fmt = None -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True -html_use_smartypants = False - # Custom sidebar templates, maps document names to template names. # html_sidebars = {} diff --git a/docs/src/docutils.conf b/docs/src/docutils.conf new file mode 100644 index 00000000..168f9e2b --- /dev/null +++ b/docs/src/docutils.conf @@ -0,0 +1,2 @@ +[parsers] +smart_quotes: false diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 90168f86..fb87ffe2 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -72,24 +72,20 @@ GENERAL ------- All commands accept the following options: -.. option:: -h, --help - +**-h, --help** Show the help screen. -.. option:: -l LOGLEVEL, --log-level LOGLEVEL - +**-l LOGLEVEL, --log-level LOGLEVEL** Set the specified verbosity level. The levels, in order from the lowest to the highest, are: ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4 or higher). If used along with -v then -v increases last set value and -l overwrites last set value. -.. option:: -q, --quiet - +**-q, --quiet** Quiet mode: disables logging, including WARNING and ERROR. -.. option:: -v, --verbose - +**-v, --verbose** Increase the verbosity level. Every instance of -v increments the verbosity level by one. Its default value is 0 which includes ERROR and WARNING levels. @@ -99,8 +95,7 @@ All commands accept the following options: then -l overwrites last set value and -v increases last set value. -.. option:: -V, --version - +**-V, --version** Show version and exit. @@ -115,55 +110,45 @@ CONFIG/INSTALL Configure/install one or more hosts. Install command is currently in beta. -.. option:: -4, --force-ipv4 - +**-4, --force-ipv4** Force to use IPv4 addresses only. No influence for custom remote commands. -.. option:: -6, --force-ipv6 - +**-6, --force-ipv6** Force to use IPv6 addresses only. No influence for custom remote commands. -.. option:: -A, --all-tagged - +**-A, --all-tagged** Use all hosts present in tags db. Currently in beta. -.. option:: -a, --all - +**-a, --all** List hosts that have all specified tags, if -t/--tag is specified. -.. option:: -b, --beta - +**-b, --beta** Enable beta functionality. -.. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN - +**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** Sepcify custom cache path pattern. If it is not set then default hostdir is used. For more info on format see :strong:`CACHE PATH PATTERN FORMAT` below. -.. option:: -c CONF_DIR, --conf-dir CONF_DIR - +**-c CONF_DIR, --conf-dir CONF_DIR** Add a configuration directory. Can be specified multiple times. If configuration directories contain conflicting types, explorers or manifests, then the last one found is used. -.. option:: -f HOSTFILE, --file HOSTFILE - +**-f HOSTFILE, --file HOSTFILE** Read specified file for a list of additional hosts to operate on or if '-' is given, read stdin (one host per line). If no host or host file is specified then, by default, read hosts from stdin. For the file format see :strong:`HOSTFILE FORMAT` below. -.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE - +**-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. -.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR - +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory directory is set up by the following rules: if cdist configuration resolves this value then specified @@ -171,60 +156,48 @@ Install command is currently in beta. ~/.cdit/inventory is used, otherwise distribution inventory directory is used. -.. option:: -i MANIFEST, --initial-manifest MANIFEST - +**-i MANIFEST, --initial-manifest MANIFEST** Path to a cdist manifest or - to read from stdin. -.. option:: -j [JOBS], --jobs [JOBS] - +**-j [JOBS], --jobs [JOBS]** Operate in parallel in specified maximum number of jobs. Global explorers, object prepare and object run are supported. Without argument CPU count is used by default. Currently in beta. -.. option:: -n, --dry-run - +**-n, --dry-run** Do not execute code. -.. option:: -o OUT_PATH, --out-dir OUT_PATH - +**-o OUT_PATH, --out-dir OUT_PATH** Directory to save cdist output in. -.. option:: -p [HOST_MAX], --parallel [HOST_MAX] - +**-p [HOST_MAX], --parallel [HOST_MAX]** Operate on multiple hosts in parallel for specified maximum hosts at a time. Without argument CPU count is used by default. -.. option:: -R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}] - +**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]** Operate by using archiving with compression where appropriate. Supported values are: tar - tar archive, tgz - gzip tar archive (the default), tbz2 - bzip2 tar archive and txz - lzma tar archive. Currently in beta. -.. option:: -r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH - +**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** Directory to save cdist output in on the target host. -.. option:: -S, --disable-saving-output-streams - +**-S, --disable-saving-output-streams** Disable saving output streams. -.. option:: -s, --sequential - +**-s, --sequential** Operate on multiple hosts sequentially (default). -.. option:: --remote-copy REMOTE_COPY - +**--remote-copy REMOTE_COPY** Command to use for remote copy (should behave like scp). -.. option:: --remote-exec REMOTE_EXEC - +**--remote-exec REMOTE_EXEC** Command to use for remote execution (should behave like ssh). -.. option:: -t, --tag - +**-t, --tag** Host is specified by tag, not hostname/address; list all hosts that contain any of specified tags. Currently in beta. @@ -271,27 +244,22 @@ INVENTORY ADD-HOST ------------------ Add host(s) to inventory database. -.. option:: host - +**host** Host(s) to add. -.. option:: -b, --beta - +**-b, --beta** Enable beta functionality. -.. option:: -f HOSTFILE, --file HOSTFILE - +**-f HOSTFILE, --file HOSTFILE** Read additional hosts to add from specified file or from stdin if '-' (each host on separate line). If no host or host file is specified then, by default, read from stdin. Hostfile format is the same as config hostfile format. -.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE - +**-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. -.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR - +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory directory is set up by the following rules: if cdist configuration resolves this value then specified @@ -304,16 +272,13 @@ INVENTORY ADD-TAG ----------------- Add tag(s) to inventory database. -.. option:: host - +**host** List of host(s) for which tags are added. -.. option:: -b, --beta - +**-b, --beta** Enable beta functionality. -.. option:: -f HOSTFILE, --file HOSTFILE - +**-f HOSTFILE, --file HOSTFILE** Read additional hosts to add tags from specified file or from stdin if '-' (each host on separate line). If no host or host file is specified then, by default, @@ -321,12 +286,10 @@ Add tag(s) to inventory database. are specified then tags are read from stdin and are added to all hosts. Hostfile format is the same as config hostfile format. -.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE - +**-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. -.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR - +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory directory is set up by the following rules: if cdist configuration resolves this value then specified @@ -334,8 +297,7 @@ Add tag(s) to inventory database. ~/.cdit/inventory is used, otherwise distribution inventory directory is used. -.. option:: -T TAGFILE, --tag-file TAGFILE - +**-T TAGFILE, --tag-file TAGFILE** Read additional tags to add from specified file or from stdin if '-' (each tag on separate line). If no tag or tag file is specified then, by default, read @@ -343,8 +305,7 @@ Add tag(s) to inventory database. specified then tags are read from stdin and are added to all hosts. Tagfile format is the same as config hostfile format. -.. option:: -t TAGLIST, --taglist TAGLIST - +**-t TAGLIST, --taglist TAGLIST** Tag list to be added for specified host(s), comma separated values. @@ -353,31 +314,25 @@ INVENTORY DEL-HOST ------------------ Delete host(s) from inventory database. -.. option:: host - +**host** Host(s) to delete. -.. option:: -a, --all - +**-a, --all** Delete all hosts. -.. option:: -b, --beta - +**-b, --beta** Enable beta functionality. -.. option:: -f HOSTFILE, --file HOSTFILE - +**-f HOSTFILE, --file HOSTFILE** Read additional hosts to delete from specified file or from stdin if '-' (each host on separate line). If no host or host file is specified then, by default, read from stdin. Hostfile format is the same as config hostfile format. -.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE - +**-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. -.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR - +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory directory is set up by the following rules: if cdist configuration resolves this value then specified @@ -390,20 +345,16 @@ INVENTORY DEL-TAG ----------------- Delete tag(s) from inventory database. -.. option:: host - +**host** List of host(s) for which tags are deleted. -.. option:: -a, --all - +**-a, --all** Delete all tags for specified host(s). -.. option:: -b, --beta - +**-b, --beta** Enable beta functionality. -.. option:: -f HOSTFILE, --file HOSTFILE - +**-f HOSTFILE, --file HOSTFILE** Read additional hosts to delete tags for from specified file or from stdin if '-' (each host on separate line). If no host or host file is specified @@ -412,12 +363,10 @@ Delete tag(s) from inventory database. from stdin and are deleted from all hosts. Hostfile format is the same as config hostfile format. -.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE - +**-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. -.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR - +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory directory is set up by the following rules: if cdist configuration resolves this value then specified @@ -425,8 +374,7 @@ Delete tag(s) from inventory database. ~/.cdit/inventory is used, otherwise distribution inventory directory is used. -.. option:: -T TAGFILE, --tag-file TAGFILE - +**-T TAGFILE, --tag-file TAGFILE** Read additional tags from specified file or from stdin if '-' (each tag on separate line). If no tag or tag file is specified then, by default, read from stdin. @@ -434,8 +382,7 @@ Delete tag(s) from inventory database. then tags are read from stdin and are added to all hosts. Tagfile format is the same as config hostfile format. -.. option:: -t TAGLIST, --taglist TAGLIST - +**-t TAGLIST, --taglist TAGLIST** Tag list to be deleted for specified host(s), comma separated values. @@ -444,36 +391,29 @@ INVENTORY LIST -------------- List inventory database. -.. option:: host - +**host** Host(s) to list. -.. option:: -a, --all - +**-a, --all** List hosts that have all specified tags, if -t/--tag is specified. -.. option:: -b, --beta - +**-b, --beta** Enable beta functionality. -.. option:: -f HOSTFILE, --file HOSTFILE - +**-f HOSTFILE, --file HOSTFILE** Read additional hosts to list from specified file or from stdin if '-' (each host on separate line). If no host or host file is specified then, by default, list all. Hostfile format is the same as config hostfile format. -.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE - +**-g CONFIG_FILE, --config-file CONFIG_FILE** Use specified custom configuration file. -.. option:: -H, --host-only - +**-H, --host-only** Suppress tags listing. -.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR - +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory directory is set up by the following rules: if cdist configuration resolves this value then specified @@ -481,8 +421,7 @@ List inventory database. ~/.cdit/inventory is used, otherwise distribution inventory directory is used. -.. option:: -t, --tag - +**-t, --tag** Host is specified by tag, not hostname/address; list all hosts that contain any of specified tags. @@ -494,8 +433,7 @@ to the types as commands. It can be thought as an "interactive manifest" environment. See below for example usage. Its primary use is for debugging type parameters. -.. option:: -s SHELL, --shell SHELL - +**-s SHELL, --shell SHELL** Select shell to use, defaults to current shell. Used shell should be POSIX compatible shell. From 7de80678f02bdc2badf94923c5a68ea28cd33a13 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 16 Mar 2018 19:17:53 +0100 Subject: [PATCH 026/178] Release 4.8.3 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 64ab1157..7348974e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.8.3: 2018-03-16 * Type __key_value: Add onchange parameter (Kamila Součková) * Types __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) From 958222133bf4d369c28b09636520d5af83b8080d Mon Sep 17 00:00:00 2001 From: Dmitry Bogatov Date: Sun, 18 Mar 2018 20:38:44 +0300 Subject: [PATCH 027/178] Fix spelling errors in manpages --- cdist/conf/type/__docker_swarm/man.rst | 2 +- docs/src/cdist-inventory.rst | 2 +- docs/src/man1/cdist.rst | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__docker_swarm/man.rst b/cdist/conf/type/__docker_swarm/man.rst index 7b0a57d3..4dc408f0 100644 --- a/cdist/conf/type/__docker_swarm/man.rst +++ b/cdist/conf/type/__docker_swarm/man.rst @@ -28,7 +28,7 @@ EXAMPLES .. code-block:: sh - # Initalizes a swarm + # Initializes a swarm __docker_swarm # Leaves a swarm diff --git a/docs/src/cdist-inventory.rst b/docs/src/cdist-inventory.rst index 584fe310..106fcdb6 100644 --- a/docs/src/cdist-inventory.rst +++ b/docs/src/cdist-inventory.rst @@ -64,7 +64,7 @@ Examples # Delete hosts from file old-hosts from inventory $ cdist inventory del-host -b -f old-hosts - # Add tags to specifed hosts + # Add tags to specified hosts $ cdist inventory add-tag -b -t europe,croatia,web,static web1 web2 # Add tag to all hosts in inventory diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index fb87ffe2..79683883 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -129,7 +129,7 @@ Install command is currently in beta. Enable beta functionality. **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** - Sepcify custom cache path pattern. If it is not set then + Specify custom cache path pattern. If it is not set then default hostdir is used. For more info on format see :strong:`CACHE PATH PATTERN FORMAT` below. @@ -604,7 +604,7 @@ EXAMPLES # Delete hosts from file old-hosts from inventory % cdist inventory del-host -b -f old-hosts - # Add tags to specifed hosts + # Add tags to specified hosts % cdist inventory add-tag -b -t europe,croatia,web,static web1 web2 # Add tag to all hosts in inventory From b740d415cad11a5e3a1d3f1be26983e3f3bd5bb7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 19 Mar 2018 15:18:08 +0100 Subject: [PATCH 028/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 7348974e..272d1751 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Documentation, type manpages: Fix spelling (Dmitry Bogatov) + 4.8.3: 2018-03-16 * Type __key_value: Add onchange parameter (Kamila Součková) * Types __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: From b695e82becffc0c13243b517df05ab9a7d1d3d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamila=20Sou=C4=8Dkov=C3=A1?= Date: Mon, 26 Mar 2018 08:53:31 +0200 Subject: [PATCH 029/178] FreeBSD improvements (#651) --- cdist/conf/explorer/is-freebsd-jail | 1 + cdist/conf/type/__hostname/gencode-remote | 4 +-- cdist/conf/type/__hostname/manifest | 7 +++++ .../conf/type/__start_on_boot/explorer/state | 4 +++ .../conf/type/__start_on_boot/gencode-remote | 4 +++ cdist/conf/type/__start_on_boot/manifest | 28 +++++++++++++++++++ cdist/conf/type/__sysctl/gencode-remote | 12 +++++++- cdist/conf/type/__sysctl/manifest | 2 +- 8 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 cdist/conf/explorer/is-freebsd-jail create mode 100644 cdist/conf/type/__start_on_boot/manifest diff --git a/cdist/conf/explorer/is-freebsd-jail b/cdist/conf/explorer/is-freebsd-jail new file mode 100644 index 00000000..a6d11d1a --- /dev/null +++ b/cdist/conf/explorer/is-freebsd-jail @@ -0,0 +1 @@ +sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index dbffad61..89bf7b3f 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -40,7 +40,7 @@ case "$os" in exit 0 fi ;; - scientific|centos|openbsd) + scientific|centos|freebsd|openbsd) if [ "$name_sysconfig" = "$name_should" -a "$name_running" = "$name_should" ]; then exit 0 fi @@ -62,7 +62,7 @@ case "$os" in echo "printf '%s\n' '$name_should' > /etc/hostname" echo "hostname -F /etc/hostname" ;; - openbsd) + freebsd|openbsd) echo "hostname '$name_should'" ;; suse) diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index 4836c501..74664aa7 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -52,6 +52,13 @@ case "$os" in --key HOSTNAME \ --value "$name_should" --exact_delimiter ;; + freebsd) + __key_value rcconf-hostname \ + --file /etc/rc.conf \ + --delimiter '=' \ + --key 'hostname' \ + --value "$name_should" + ;; openbsd) echo "$name_should" | __file /etc/myname --source - ;; diff --git a/cdist/conf/type/__start_on_boot/explorer/state b/cdist/conf/type/__start_on_boot/explorer/state index d49f01c7..d8f9b7ba 100644 --- a/cdist/conf/type/__start_on_boot/explorer/state +++ b/cdist/conf/type/__start_on_boot/explorer/state @@ -64,6 +64,10 @@ else state="present" [ -f "/etc/runlevels/${target_runlevel}/${name}" ] || state="absent" ;; + freebsd) + state="absent" + service -e | grep "/$name$" && state="present" + ;; *) echo "Unsupported os: $os" >&2 exit 1 diff --git a/cdist/conf/type/__start_on_boot/gencode-remote b/cdist/conf/type/__start_on_boot/gencode-remote index 63f0ba3c..14ee7dab 100755 --- a/cdist/conf/type/__start_on_boot/gencode-remote +++ b/cdist/conf/type/__start_on_boot/gencode-remote @@ -77,6 +77,10 @@ case "$state_should" in echo "update-rc.d \"$name\" defaults >/dev/null" ;; + freebsd) + : # handled in manifest + ;; + *) echo "Unsupported os: $os" >&2 exit 1 diff --git a/cdist/conf/type/__start_on_boot/manifest b/cdist/conf/type/__start_on_boot/manifest new file mode 100644 index 00000000..b9ee20e2 --- /dev/null +++ b/cdist/conf/type/__start_on_boot/manifest @@ -0,0 +1,28 @@ +#!/bin/sh -e + +state_should="$(cat "$__object/parameter/state")" +state_is=$(cat "$__object/explorer/state") +name="$__object_id" + +# Short circuit if nothing is to be done +[ "$state_should" = "$state_is" ] && exit 0 + +os=$(cat "$__global/explorer/os") + +case "$os" in + freebsd) + if [ "$state_should" = 'present' ]; then + value='YES' + else + value='NO' + fi + __key_value rcconf-$name-enable \ + --file /etc/rc.conf \ + --key "${name}_enable" \ + --value "\"$value\"" \ + --delimiter '=' + ;; + *) + : # handled in gencode-remote + ;; +esac diff --git a/cdist/conf/type/__sysctl/gencode-remote b/cdist/conf/type/__sysctl/gencode-remote index b7fb02c8..a7aedb84 100755 --- a/cdist/conf/type/__sysctl/gencode-remote +++ b/cdist/conf/type/__sysctl/gencode-remote @@ -26,5 +26,15 @@ if [ "$value_should" = "$value_is" ]; then exit 0 fi +os=$(cat "$__global/explorer/os") +case "$os" in + redhat|centos|ubuntu|debian|devuan|archlinux|coreos) + flag='-w' + ;; + frebsd) + flag='' + ;; +esac + # set the current runtime value -printf 'sysctl -w %s="%s"\n' "$__object_id" "$value_should" +printf 'sysctl %s %s="%s"\n' "$flag" "$__object_id" "$value_should" diff --git a/cdist/conf/type/__sysctl/manifest b/cdist/conf/type/__sysctl/manifest index 39a2e53c..c903dbae 100755 --- a/cdist/conf/type/__sysctl/manifest +++ b/cdist/conf/type/__sysctl/manifest @@ -22,7 +22,7 @@ os=$(cat "$__global/explorer/os") case "$os" in - redhat|centos|ubuntu|debian|devuan|archlinux|coreos) + redhat|centos|ubuntu|debian|devuan|archlinux|coreos|freebsd) : ;; *) From 60eba4214137f2eb43052066cae8e585629603d4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 26 Mar 2018 08:52:41 +0200 Subject: [PATCH 030/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 272d1751..ce5a169d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * Documentation, type manpages: Fix spelling (Dmitry Bogatov) + * New explorer: is-freebsd-jail (Kamila Součková) + * Types __hostname, __start_on_boot, __sysctl: Support FreeBSD (Kamila Součková) 4.8.3: 2018-03-16 * Type __key_value: Add onchange parameter (Kamila Součková) From 999f221c880f2209c178627c892c5ec3fefebfb6 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 5 Apr 2018 10:32:33 +0200 Subject: [PATCH 031/178] need a way to distinguish between install-config and normal config Signed-off-by: Steven Armstrong --- cdist/conf/type/__install_config/gencode-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__install_config/gencode-local b/cdist/conf/type/__install_config/gencode-local index 8f24cf2e..dd4f2a78 100755 --- a/cdist/conf/type/__install_config/gencode-local +++ b/cdist/conf/type/__install_config/gencode-local @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -23,6 +23,7 @@ remote_exec="$__type/files/remote/exec" remote_copy="$__type/files/remote/copy" cat << DONE +export __cdist_install_config=yes export __cdist_log_level=$__cdist_log_level export __default_remote_exec="$__remote_exec" export __default_remote_copy="$__remote_copy" From 43a65d91b10cb2e393bf74e74a09f0e597992f32 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 5 Apr 2018 10:36:38 +0200 Subject: [PATCH 032/178] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index ce5a169d..bd8996cb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,8 @@ next: * Documentation, type manpages: Fix spelling (Dmitry Bogatov) * New explorer: is-freebsd-jail (Kamila Součková) * Types __hostname, __start_on_boot, __sysctl: Support FreeBSD (Kamila Součková) + * Type __install_config: set environment variable to distinguish between + install-config and regular config (Steven Armstrong) 4.8.3: 2018-03-16 * Type __key_value: Add onchange parameter (Kamila Součková) From f09765a03af9fe9576acff4041b342d2ffb6324d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 17 Apr 2018 11:15:28 +0200 Subject: [PATCH 033/178] Improve error reporting. --- cdist/__init__.py | 81 +++++++++++++++++++++++++++++++++----------- cdist/config.py | 3 +- cdist/exec/local.py | 6 ++-- cdist/exec/remote.py | 6 ++-- cdist/exec/util.py | 1 + 5 files changed, 68 insertions(+), 29 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index e6fdfac6..b1b9df9f 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -83,41 +83,72 @@ class CdistBetaRequired(cdist.Error): class CdistEntityError(Error): """Something went wrong while executing cdist entity""" - def __init__(self, entity_name, entity_params, stderr_paths, subject=''): + def __init__(self, entity_name, entity_params, stdout_paths, + stderr_paths, subject=''): self.entity_name = entity_name self.entity_params = entity_params self.stderr_paths = stderr_paths + self.stdout_paths = stdout_paths if isinstance(subject, Error): self.original_error = subject else: self.original_error = None self.message = str(subject) - @property - def stderr(self): - output = [] - for stderr_name, stderr_path in self.stderr_paths: - if (os.path.exists(stderr_path) and - os.path.getsize(stderr_path) > 0): - label_begin = '---- BEGIN ' + stderr_name + ':stderr ----' - label_end = '---- END ' + stderr_name + ':stderr ----' - output.append('\n' + label_begin) - with open(stderr_path, 'r') as fd: + def _stdpath(self, stdpaths, header_name): + result = {} + for name, path in stdpaths: + if name not in result: + result[name] = [] + if os.path.exists(path) and os.path.getsize(path) > 0: + output = [] + label_begin = name + ":" + header_name + output.append(label_begin) + output.append('\n') + output.append('-' * len(label_begin)) + output.append('\n') + with open(path, 'r') as fd: output.append(fd.read()) - output.append(label_end) - return '\n'.join(output) + output.append('\n') + result[name].append(''.join(output)) + return result + + def _stderr(self): + return self._stdpath(self.stderr_paths, 'stderr') + + def _stdout(self): + return self._stdpath(self.stdout_paths, 'stdout') + + def _update_dict_list(self, target, source): + for x in source: + if x not in target: + target[x] = [] + target[x].extend(source[x]) + + @property + def std_streams(self): + std_dict = {} + self._update_dict_list(std_dict, self._stdout()) + self._update_dict_list(std_dict, self._stderr()) + return std_dict def __str__(self): output = [] output.append(self.message) - header = "\nError processing " + self.entity_name + output.append('\n\n') + header = "Error processing " + self.entity_name under_header = '=' * len(header) output.append(header) + output.append('\n') output.append(under_header) + output.append('\n') for param_name, param_value in self.entity_params: output.append(param_name + ': ' + str(param_value)) - output.append(self.stderr + '\n') - return '\n'.join(output) + output.append('\n') + output.append('\n') + for x in self.std_streams: + output.append(''.join(self.std_streams[x])) + return ''.join(output) class CdistObjectError(CdistEntityError): @@ -127,28 +158,38 @@ class CdistObjectError(CdistEntityError): ('name', cdist_object.name, ), ('path', cdist_object.absolute_path, ), ('source', " ".join(cdist_object.source), ), - ('type', cdist_object.cdist_type.absolute_path, ), + ('type', os.path.realpath(cdist_object.cdist_type.absolute_path), ), ] stderr_paths = [] for stderr_name in os.listdir(cdist_object.stderr_path): stderr_path = os.path.join(cdist_object.stderr_path, stderr_name) stderr_paths.append((stderr_name, stderr_path, )) + stdout_paths = [] + for stdout_name in os.listdir(cdist_object.stdout_path): + stdout_path = os.path.join(cdist_object.stdout_path, + stdout_name) + stdout_paths.append((stdout_name, stdout_path, )) super().__init__("object '{}'".format(cdist_object.name), - params, stderr_paths, subject) + params, stdout_paths, stderr_paths, subject) class InitialManifestError(CdistEntityError): """Something went wrong while executing initial manifest""" - def __init__(self, initial_manifest, stderr_path, subject=''): + def __init__(self, initial_manifest, stdout_path, stderr_path, subject=''): params = [ ('path', initial_manifest, ), ] + stdout_paths = [] + stdout_paths = [ + ('init', stdout_path, ), + ] stderr_paths = [] stderr_paths = [ ('init', stderr_path, ), ] - super().__init__('initial manifest', params, stderr_paths, subject) + super().__init__('initial manifest', params, stdout_paths, + stderr_paths, subject) def file_to_list(filename): diff --git a/cdist/config.py b/cdist/config.py index 74f68a72..2dcb1005 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -440,9 +440,10 @@ class Config(object): self.manifest.run_initial_manifest(self.local.initial_manifest) except cdist.Error as e: which = "init" + stdout_path = os.path.join(self.local.stdout_base_path, which) stderr_path = os.path.join(self.local.stderr_base_path, which) raise cdist.InitialManifestError(self.local.initial_manifest, - stderr_path, e) + stdout_path, stderr_path, e) self.iterate_until_finished() self.cleanup() self._remove_files_dirs() diff --git a/cdist/exec/local.py b/cdist/exec/local.py index a50fe072..f83c85df 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -259,10 +259,8 @@ class Local(object): util.log_std_fd(self.log, command, stderr, 'Local stderr') util.log_std_fd(self.log, command, stdout, 'Local stdout') return output - except subprocess.CalledProcessError as e: - util.handle_called_process_error(e, command) - except OSError as error: - raise cdist.Error(" ".join(command) + ": " + error.args[1]) + except (OSError, subprocess.CalledProcessError) as error: + raise cdist.Error(" ".join(command) + ": " + str(error.args[1])) finally: if message_prefix: message.merge_messages() diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index b75905ba..4466545e 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -343,10 +343,8 @@ class Remote(object): util.log_std_fd(self.log, command, stdout, 'Remote stdout') return output - except subprocess.CalledProcessError as e: - util.handle_called_process_error(e, command) - except OSError as error: - raise cdist.Error(" ".join(command) + ": " + error.args[1]) + except (OSError, subprocess.CalledProcessError) as error: + raise cdist.Error(" ".join(command) + ": " + str(error.args[1])) except UnicodeDecodeError: raise DecodeError(command) finally: diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 2f2aa38c..c96f757b 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -127,6 +127,7 @@ def call_get_output(command, env=None, stderr=None): return (_call_get_stdout(command, env, stderr), None) +# Currently not used. def handle_called_process_error(err, command): # Currently, stderr is not captured. # errout = None From 2a9057f892f67014a67a34ffb260318ea8465349 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 19 Apr 2018 17:53:47 +0200 Subject: [PATCH 034/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bd8996cb..e958ddff 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Types __hostname, __start_on_boot, __sysctl: Support FreeBSD (Kamila Součková) * Type __install_config: set environment variable to distinguish between install-config and regular config (Steven Armstrong) + * Core: Improve error reporting (Darko Poljak) 4.8.3: 2018-03-16 * Type __key_value: Add onchange parameter (Kamila Součková) From 7349da777be5316a9130033af7f9ebf852ddb02d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Apr 2018 14:24:51 +0200 Subject: [PATCH 035/178] ++changelog --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index e958ddff..681256fc 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,7 +6,7 @@ next: * New explorer: is-freebsd-jail (Kamila Součková) * Types __hostname, __start_on_boot, __sysctl: Support FreeBSD (Kamila Součková) * Type __install_config: set environment variable to distinguish between - install-config and regular config (Steven Armstrong) + install-config and regular config (Steven Armstrong) * Core: Improve error reporting (Darko Poljak) 4.8.3: 2018-03-16 From 09642142f478ee61b804f7ad1f6f5ecd6fd70d66 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Apr 2018 14:31:10 +0200 Subject: [PATCH 036/178] Release 4.8.4 --- cdist/__init__.py | 3 ++- docs/changelog | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index b1b9df9f..b7602ff8 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -158,7 +158,8 @@ class CdistObjectError(CdistEntityError): ('name', cdist_object.name, ), ('path', cdist_object.absolute_path, ), ('source', " ".join(cdist_object.source), ), - ('type', os.path.realpath(cdist_object.cdist_type.absolute_path), ), + ('type', os.path.realpath( + cdist_object.cdist_type.absolute_path), ), ] stderr_paths = [] for stderr_name in os.listdir(cdist_object.stderr_path): diff --git a/docs/changelog b/docs/changelog index 681256fc..6330a149 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.8.4: 2018-04-20 * Documentation, type manpages: Fix spelling (Dmitry Bogatov) * New explorer: is-freebsd-jail (Kamila Součková) * Types __hostname, __start_on_boot, __sysctl: Support FreeBSD (Kamila Součková) From aa20ed5ea1b3c833acc78c54166f655c2b9cd355 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Sun, 29 Apr 2018 13:02:18 +0200 Subject: [PATCH 037/178] __docker_stack: Use --with-registry-auth deploy option Without this option, Swarm agents are unable to download images from private registries. --- cdist/conf/type/__docker_stack/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_stack/gencode-remote b/cdist/conf/type/__docker_stack/gencode-remote index 0d47dc52..586271d0 100755 --- a/cdist/conf/type/__docker_stack/gencode-remote +++ b/cdist/conf/type/__docker_stack/gencode-remote @@ -50,7 +50,7 @@ case "${state}" in eof docker stack deploy --compose-file "\${compose_file}" \ - --prune ${stack} + --prune --with-registry-auth ${stack} rm "\${compose_file}" EOF From 3869940fc8ae4a8c35656298ce40146a111b7064 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 29 Apr 2018 20:39:39 +0200 Subject: [PATCH 038/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 6330a149..74d6d7c6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) + 4.8.4: 2018-04-20 * Documentation, type manpages: Fix spelling (Dmitry Bogatov) * New explorer: is-freebsd-jail (Kamila Součková) From 4f61f6be9cd2052ede37dfb184211e23e9ec6328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Mon, 30 Apr 2018 12:50:09 +0200 Subject: [PATCH 039/178] Add new type __docker_config --- .../type/__docker_config/explorer/config-data | 21 ++++++ .../__docker_config/explorer/config-exists | 25 +++++++ .../conf/type/__docker_config/gencode-remote | 69 +++++++++++++++++++ cdist/conf/type/__docker_config/man.rst | 55 +++++++++++++++ .../__docker_config/parameter/default/source | 0 .../__docker_config/parameter/default/state | 1 + .../type/__docker_config/parameter/optional | 2 + 7 files changed, 173 insertions(+) create mode 100644 cdist/conf/type/__docker_config/explorer/config-data create mode 100755 cdist/conf/type/__docker_config/explorer/config-exists create mode 100755 cdist/conf/type/__docker_config/gencode-remote create mode 100644 cdist/conf/type/__docker_config/man.rst create mode 100644 cdist/conf/type/__docker_config/parameter/default/source create mode 100644 cdist/conf/type/__docker_config/parameter/default/state create mode 100644 cdist/conf/type/__docker_config/parameter/optional diff --git a/cdist/conf/type/__docker_config/explorer/config-data b/cdist/conf/type/__docker_config/explorer/config-data new file mode 100644 index 00000000..926a795d --- /dev/null +++ b/cdist/conf/type/__docker_config/explorer/config-data @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +docker config inspect "${__object_id:?}" | jq -r '.[0].Spec.Data' | base64 -d diff --git a/cdist/conf/type/__docker_config/explorer/config-exists b/cdist/conf/type/__docker_config/explorer/config-exists new file mode 100755 index 00000000..58c207d4 --- /dev/null +++ b/cdist/conf/type/__docker_config/explorer/config-exists @@ -0,0 +1,25 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if docker config ls | grep -q " ${__object_id:?} "; then + echo yes +else + echo no +fi diff --git a/cdist/conf/type/__docker_config/gencode-remote b/cdist/conf/type/__docker_config/gencode-remote new file mode 100755 index 00000000..65497b7e --- /dev/null +++ b/cdist/conf/type/__docker_config/gencode-remote @@ -0,0 +1,69 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +config="${__object_id:?}" +config_exists=$(cat "${__object:?}/explorer/config-exists") +state=$(cat "${__object:?}/parameter/state") + +case "${state}" in + absent) + if [ "${config_exists}" != "yes" ]; then + exit 0 + fi + + echo "docker config rm \"${config}\"" + ;; + present) + source=$(cat "${__object}/parameter/source") + + if [ -z "${source}" ]; then + exit 0 + fi + + if [ "${source}" = "-" ]; then + source="${__object}/stdin" + fi + + if [ "${config_exists}" = "yes" ]; then + if cmp -s "${source}" "${__object}/explorer/config-data"; then + exit 0 + else + echo "docker config rm \"${config}\"" + fi + fi + + cat <<-EOF + source_file="\$(mktemp cdist.XXXXXXXXXX)" + + base64 -d > "\${source_file}" << eof + $(base64 "${source}") + eof + + docker config create "${config}" "\${source_file}" + + rm "\${source_file}" + EOF + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_config/man.rst b/cdist/conf/type/__docker_config/man.rst new file mode 100644 index 00000000..7c74c8af --- /dev/null +++ b/cdist/conf/type/__docker_config/man.rst @@ -0,0 +1,55 @@ +cdist-type__docker_config(7) +============================ + +NAME +---- + +cdist-type__docker_config - Manage Docker configs + +DESCRIPTION +----------- + +This type manages Docker configs. + +OPTIONAL PARAMETERS +------------------- + +source + Path to the source file. If it is '-' (dash), read standard input. + +state + 'present' or 'absent', defaults to 'present' where: + + present + if the config does not exist, it is created + absent + the config is removed + +CAVEATS +------- + +Since Docker configs cannot be updated once created, this type tries removing +and recreating the config if it changes. If the config is used by a service at +the time of removing, then this type will fail. + +EXAMPLES +-------- + +.. code-block:: sh + + # Creates "foo" config from "bar" source file + __docker_config foo --source bar + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_config/parameter/default/source b/cdist/conf/type/__docker_config/parameter/default/source new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__docker_config/parameter/default/state b/cdist/conf/type/__docker_config/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_config/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_config/parameter/optional b/cdist/conf/type/__docker_config/parameter/optional new file mode 100644 index 00000000..d77f3048 --- /dev/null +++ b/cdist/conf/type/__docker_config/parameter/optional @@ -0,0 +1,2 @@ +source +state From 5a7d74b90542de9ad179268c0561fbffbf0c8f1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Mon, 30 Apr 2018 12:52:54 +0200 Subject: [PATCH 040/178] Add new type __docker_secret --- .../__docker_secret/explorer/secret-exists | 25 +++++++ .../conf/type/__docker_secret/gencode-remote | 65 +++++++++++++++++++ cdist/conf/type/__docker_secret/man.rst | 54 +++++++++++++++ .../__docker_secret/parameter/default/source | 0 .../__docker_secret/parameter/default/state | 1 + .../type/__docker_secret/parameter/optional | 2 + 6 files changed, 147 insertions(+) create mode 100644 cdist/conf/type/__docker_secret/explorer/secret-exists create mode 100644 cdist/conf/type/__docker_secret/gencode-remote create mode 100644 cdist/conf/type/__docker_secret/man.rst create mode 100644 cdist/conf/type/__docker_secret/parameter/default/source create mode 100644 cdist/conf/type/__docker_secret/parameter/default/state create mode 100644 cdist/conf/type/__docker_secret/parameter/optional diff --git a/cdist/conf/type/__docker_secret/explorer/secret-exists b/cdist/conf/type/__docker_secret/explorer/secret-exists new file mode 100644 index 00000000..1405f8bc --- /dev/null +++ b/cdist/conf/type/__docker_secret/explorer/secret-exists @@ -0,0 +1,25 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if docker secret ls | grep -q " ${__object_id:?} "; then + echo yes +else + echo no +fi diff --git a/cdist/conf/type/__docker_secret/gencode-remote b/cdist/conf/type/__docker_secret/gencode-remote new file mode 100644 index 00000000..c75e91d9 --- /dev/null +++ b/cdist/conf/type/__docker_secret/gencode-remote @@ -0,0 +1,65 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +secret="${__object_id:?}" +secret_exists=$(cat "${__object:?}/explorer/secret-exists") +state=$(cat "${__object:?}/parameter/state") + +case "${state}" in + absent) + if [ "${secret_exists}" != "yes" ]; then + exit 0 + fi + + echo "docker secret rm ${secret}" + ;; + present) + if [ "${secret_exists}" = "yes" ]; then + exit 0 + fi + + source=$(cat "${__object}/parameter/source") + + if [ -z "${source}" ]; then + exit 0 + fi + + if [ "${source}" = "-" ]; then + source="${__object}/stdin" + fi + + cat <<-EOF + source_file="\$(mktemp cdist.XXXXXXXXXX)" + + base64 -d > "\${source_file}" << eof + $(base64 "${source}") + eof + + docker secret create "${secret}" "\${source_file}" + + rm "\${source_file}" + EOF + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_secret/man.rst b/cdist/conf/type/__docker_secret/man.rst new file mode 100644 index 00000000..7fe69623 --- /dev/null +++ b/cdist/conf/type/__docker_secret/man.rst @@ -0,0 +1,54 @@ +cdist-type__docker_secret(7) +============================ + +NAME +---- + +cdist-type__docker_secret - Manage Docker secrets + +DESCRIPTION +----------- + +This type manages Docker secrets. + +OPTIONAL PARAMETERS +------------------- + +source + Path to the source file. If it is '-' (dash), read standard input. + +state + 'present' or 'absent', defaults to 'present' where: + + present + if the secret does not exist, it is created + absent + the secret is removed + +CAVEATS +------- + +Since Docker secrets cannot be updated once created, this type takes no action +if the specified secret already exists. + +EXAMPLES +-------- + +.. code-block:: sh + + # Creates "foo" secret from "bar" source file + __docker_secret foo --source bar + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_secret/parameter/default/source b/cdist/conf/type/__docker_secret/parameter/default/source new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__docker_secret/parameter/default/state b/cdist/conf/type/__docker_secret/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_secret/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_secret/parameter/optional b/cdist/conf/type/__docker_secret/parameter/optional new file mode 100644 index 00000000..d77f3048 --- /dev/null +++ b/cdist/conf/type/__docker_secret/parameter/optional @@ -0,0 +1,2 @@ +source +state From b5e96efcbff20f403e5a3f4c5f3ab31c76dfde9e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 2 May 2018 08:35:12 +0200 Subject: [PATCH 041/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 74d6d7c6..3bdf446b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) + * New type: __docker_config (Ľubomír Kučera) + * New type: __docker_secret (Ľubomír Kučera) 4.8.4: 2018-04-20 * Documentation, type manpages: Fix spelling (Dmitry Bogatov) From 1efbd6a3a054d44dd4c399274fc6cd3e417e887f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Fri, 4 May 2018 16:37:45 +0200 Subject: [PATCH 042/178] __docker_config: Fix config-data explorer Before this fix, the explorer returned binary data when config did not exist. This commit also removes dependency on jq and sets executable flag on the explorer file. --- cdist/conf/type/__docker_config/explorer/config-data | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) mode change 100644 => 100755 cdist/conf/type/__docker_config/explorer/config-data diff --git a/cdist/conf/type/__docker_config/explorer/config-data b/cdist/conf/type/__docker_config/explorer/config-data old mode 100644 new mode 100755 index 926a795d..b4bb0e11 --- a/cdist/conf/type/__docker_config/explorer/config-data +++ b/cdist/conf/type/__docker_config/explorer/config-data @@ -18,4 +18,5 @@ # along with cdist. If not, see . # -docker config inspect "${__object_id:?}" | jq -r '.[0].Spec.Data' | base64 -d +docker config inspect "${__object_id:?}" --format '{{json .Spec.Data}}' \ + 2>/dev/null | tr -d '"' | base64 -d From fdb5bbcf91389626f9e32f6909db1c3bd8c89abe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Fri, 4 May 2018 16:47:09 +0200 Subject: [PATCH 043/178] __docker_secret: Set executable flag on scripts --- cdist/conf/type/__docker_secret/explorer/secret-exists | 0 cdist/conf/type/__docker_secret/gencode-remote | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 cdist/conf/type/__docker_secret/explorer/secret-exists mode change 100644 => 100755 cdist/conf/type/__docker_secret/gencode-remote diff --git a/cdist/conf/type/__docker_secret/explorer/secret-exists b/cdist/conf/type/__docker_secret/explorer/secret-exists old mode 100644 new mode 100755 diff --git a/cdist/conf/type/__docker_secret/gencode-remote b/cdist/conf/type/__docker_secret/gencode-remote old mode 100644 new mode 100755 From 22d570ae6084710d7af60dcd36ad9f0ec8ea882c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Mon, 7 May 2018 12:57:48 +0200 Subject: [PATCH 044/178] Rewrite __letsencrypt_cert type This commit adds the following features: * Ability to expand existing certificate * Ability to manage object state * Ability to obtain test certificate * Ability to promote test certificate to production * Ability to specify custom certificate name * Ability to specify multiple domains per certificate * Ability to use Certbot in standalone mode * Messaging This commit also introduces the following behavioral changes: * Attempt to install Certbot only when it is not installed already * Installation of the cron job has to be enabled using `--automatic-renewal` parameter **Note:** Object ID is now treated as certificate name and new required parameter `--domain` was added. --- .../__letsencrypt_cert/explorer/certbot-path | 3 + .../explorer/certificate-domains | 4 + .../explorer/certificate-exists | 7 + .../explorer/certificate-is-test | 8 + .../type/__letsencrypt_cert/explorer/exists | 5 - .../type/__letsencrypt_cert/gencode-remote | 85 ++++++++-- cdist/conf/type/__letsencrypt_cert/man.rst | 73 +++++++-- cdist/conf/type/__letsencrypt_cert/manifest | 152 ++++++++++-------- .../type/__letsencrypt_cert/parameter/boolean | 2 + .../parameter/default/state | 1 + .../parameter/default/webroot | 0 .../__letsencrypt_cert/parameter/optional | 2 + .../__letsencrypt_cert/parameter/required | 1 - .../parameter/required_multiple | 1 + 14 files changed, 239 insertions(+), 105 deletions(-) create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certbot-path create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists create mode 100755 cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test delete mode 100644 cdist/conf/type/__letsencrypt_cert/explorer/exists mode change 100644 => 100755 cdist/conf/type/__letsencrypt_cert/gencode-remote mode change 100644 => 100755 cdist/conf/type/__letsencrypt_cert/manifest create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/boolean create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/default/state create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/default/webroot create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/optional create mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/required_multiple diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path new file mode 100755 index 00000000..3c6076df --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path @@ -0,0 +1,3 @@ +#!/bin/sh -e + +command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains new file mode 100755 index 00000000..367fda93 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains @@ -0,0 +1,4 @@ +#!/bin/sh -e + +certbot certificates --cert-name "${__object_id:?}" | grep ' Domains: ' | \ + cut -d ' ' -f 6- | tr ' ' '\n' diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists new file mode 100755 index 00000000..d2ea35cc --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists @@ -0,0 +1,7 @@ +#!/bin/sh -e + +if certbot certificates | grep -q " Certificate Name: ${__object_id:?}$"; then + echo yes +else + echo no +fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test new file mode 100755 index 00000000..6d7b0ae9 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test @@ -0,0 +1,8 @@ +#!/bin/sh -e + +if certbot certificates --cert-name "${__object_id:?}" | \ + grep -q 'INVALID: TEST_CERT'; then + echo yes +else + echo no +fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/exists b/cdist/conf/type/__letsencrypt_cert/explorer/exists deleted file mode 100644 index cb967663..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/exists +++ /dev/null @@ -1,5 +0,0 @@ -domain=$__object_id - -if [ -f "/etc/letsencrypt/live/$domain/fullchain.pem" ]; then - echo yes -fi diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote old mode 100644 new mode 100755 index 62ada241..4bd05eff --- a/cdist/conf/type/__letsencrypt_cert/gencode-remote +++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote @@ -1,18 +1,75 @@ -domain="$__object_id" +#!/bin/sh -e -exists=$(cat "$__object/explorer/exists") -webroot="$(cat "$__object/parameter/webroot")" -admin_email="$(cat "$__object/parameter/admin-email")" +certificate_exists=$(cat "${__object:?}/explorer/certificate-exists") +name="${__object_id:?}" +state=$(cat "${__object}/parameter/state") -if [ -n "$exists" ]; then - exit 0 -fi +case "${state}" in + absent) + if [ "${certificate_exists}" = "no" ]; then + exit 0 + fi -cat <> "${__messages_out:?}" + ;; + present) + requested_domains="${__object}/parameter/domain" + + staging=no + if [ -f "${__object}/parameter/staging" ]; then + staging=yes + fi + + if [ "${certificate_exists}" = "yes" ]; then + existing_domains="${__object}/explorer/certificate-domains" + certificate_is_test=$(cat "${__object}/explorer/certificate-is-test") + + sort -uo "${requested_domains}" "${requested_domains}" + sort -uo "${existing_domains}" "${existing_domains}" + + if [ -z "$(comm -23 "${requested_domains}" "${existing_domains}")" ] && \ + [ "${certificate_is_test}" = "${staging}" ]; then + exit 0 + fi + fi + + admin_email="$(cat "$__object/parameter/admin-email")" + webroot="$(cat "$__object/parameter/webroot")" + + cat <<-EOF + certbot certonly \ + --agree-tos \ + --cert-name '${name}' \ + --email '${admin_email}' \ + --expand \ + --non-interactive \ + --quiet \ + $(if [ "${staging}" = "yes" ]; then + echo "--staging" + elif [ "${certificate_is_test}" != "${staging}" ]; then + echo "--force-renewal" + fi) \ + $(if [ -z "${webroot}" ]; then + echo "--standalone" + else + echo "--webroot --webroot-path '${webroot}'" + fi) \ + $(while read -r domain; do + echo "--domain '${domain}' \\" + done < "${requested_domains}") + EOF + + if [ "${certificate_exists}" = "no" ]; then + echo create >> "${__messages_out}" + else + echo change >> "${__messages_out}" + fi + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst index bb1e5d05..16b23fea 100644 --- a/cdist/conf/type/__letsencrypt_cert/man.rst +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -3,54 +3,95 @@ cdist-type__letsencrypt_cert(7) NAME ---- -cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt +cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt DESCRIPTION ----------- -Automatically obtain a Let's Encrypt SSL certificate. Uses certbot's webroot -method. You must set up your web server to work with webroot. +Automatically obtain a Let's Encrypt SSL certificate using Certbot. REQUIRED PARAMETERS ------------------- -webroot - The path to your webroot, as set up in your webserver config. admin-email - Where to send Let's Encrypt emails like "certificate needs renewal". + Where to send Let's Encrypt emails like "certificate needs renewal". +REQUIRED MULTIPLE PARAMETERS +---------------------------- + +domain + A domain to be included in the certificate. OPTIONAL PARAMETERS ------------------- -None. +state + 'present' or 'absent', defaults to 'present' where: + + present + if the certificate does not exist, it will be obtained + absent + the certificate will be removed + +webroot + The path to your webroot, as set up in your webserver config. If this + parameter is not present, Certbot will be run in standalone mode. OPTIONAL MULTIPLE PARAMETERS ---------------------------- + renew-hook - Renew hook command directly passed to certbot in cron job. + Renew hook command directly passed to Certbot in cron job. + +BOOLEAN PARAMETERS +------------------ + +automatic-renewal + Install a cron job, which attempts to renew certificates daily. + +staging + Obtain a test certificate from a staging server. + +MESSAGES +-------- + +change + Certificte was changed. + +create + Certificte was created. + +remove + Certificte was removed. EXAMPLES -------- .. code-block:: sh - __letsencrypt_cert example.com --admin-email root@example.com --webroot /data/letsencrypt/root - - __letsencrypt_cert example.com --admin-email root@example.com --webroot /data/letsencrypt/root --renew-hook "service nginx reload" + __letsencrypt_cert example.com \ + --admin-email root@example.com \ + --automatic-renewal \ + --domain example.com \ + --domain foo.example.com \ + --domain bar.example.com \ + --renew-hook "service nginx reload" \ + --webroot /data/letsencrypt/root AUTHORS ------- + | Nico Schottelius | Kamila Součková | Darko Poljak - +| Ľubomír Kučera COPYING ------- -Copyright \(C) 2017 Nico Schottelius, Kamila Součková, Darko Poljak. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. + +Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and +Ľubomír Kučera. You can redistribute it and/or modify it under the terms of +the GNU General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest old mode 100644 new mode 100755 index 800e5e18..94e9d225 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,79 +1,93 @@ -os=$(cat "$__global/explorer/os") -os_version=$(cat "$__global/explorer/os_version") +#!/bin/sh -case "$os" in - debian) - case "$os_version" in - 8*) - __apt_source jessie-backports \ - --uri http://http.debian.net/debian \ - --distribution jessie-backports \ - --component main +certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" - require="__apt_source/jessie-backports" __package_apt python-certbot --target-release jessie-backports - require="__apt_source/jessie-backports" __package_apt certbot --target-release jessie-backports - # Seems to be a missing dependency on debian 8 - __package python-ndg-httpsclient - ;; - 9*) - __apt_source stretch-backports \ - --uri http://http.debian.net/debian \ - --distribution stretch-backports \ - --component main +if [ -z "${certbot_fullpath}" ]; then + os="$(cat "${__global:?}/explorer/os")" + os_version="$(cat "${__global}/explorer/os_version")" - require="__apt_source/stretch-backports" __package_apt python-certbot --target-release stretch-backports - require="__apt_source/stretch-backports" __package_apt certbot --target-release stretch-backports - ;; - *) - echo "Unsupported OS version: $os_version" >&2 - exit 1 - ;; - esac + case "$os" in + debian) + case "$os_version" in + 8*) + __apt_source jessie-backports \ + --uri http://http.debian.net/debian \ + --distribution jessie-backports \ + --component main - certbot_fullpath=/usr/bin/certbot - ;; - devuan) - case "$os_version" in - jessie) - __apt_source jessie-backports \ - --uri http://auto.mirror.devuan.org/merged \ - --distribution jessie-backports \ - --component main + require="__apt_source/jessie-backports" __package_apt python-certbot \ + --target-release jessie-backports + require="__apt_source/jessie-backports" __package_apt certbot \ + --target-release jessie-backports + # Seems to be a missing dependency on debian 8 + __package python-ndg-httpsclient + ;; + 9*) + __apt_source stretch-backports \ + --uri http://http.debian.net/debian \ + --distribution stretch-backports \ + --component main - require="__apt_source/jessie-backports" __package_apt python-certbot --target-release jessie-backports - require="__apt_source/jessie-backports" __package_apt certbot --target-release jessie-backports - # Seems to be a missing dependency on debian 8 - __package python-ndg-httpsclient - ;; - *) - echo "Unsupported OS version: $os_version" >&2 - exit 1 - ;; - esac + require="__apt_source/stretch-backports" __package_apt python-certbot \ + --target-release stretch-backports + require="__apt_source/stretch-backports" __package_apt certbot \ + --target-release stretch-backports + ;; + *) + echo "Unsupported OS version: $os_version" >&2 + exit 1 + ;; + esac - certbot_fullpath=/usr/bin/certbot - ;; - freebsd) - __package py27-certbot + certbot_fullpath=/usr/bin/certbot + ;; + devuan) + case "$os_version" in + jessie) + __apt_source jessie-backports \ + --uri http://auto.mirror.devuan.org/merged \ + --distribution jessie-backports \ + --component main - certbot_fullpath=/usr/local/bin/certbot - ;; - *) - echo "Unsupported os: $os" >&2 - exit 1 - ;; -esac + require="__apt_source/jessie-backports" __package_apt python-certbot \ + --target-release jessie-backports + require="__apt_source/jessie-backports" __package_apt certbot \ + --target-release jessie-backports + # Seems to be a missing dependency on debian 8 + __package python-ndg-httpsclient + ;; + *) + echo "Unsupported OS version: $os_version" >&2 + exit 1 + ;; + esac -renew_hook_param="$__object/parameter/renew-hook" -renew_hook="" -if [ -f "$renew_hook_param" ]; then - while read hook; do - renew_hook="$renew_hook --renew-hook \"$hook\"" - done < "$renew_hook_param" + certbot_fullpath=/usr/bin/certbot + ;; + freebsd) + __package py27-certbot + + certbot_fullpath=/usr/local/bin/certbot + ;; + *) + echo "Unsupported os: $os" >&2 + exit 1 + ;; + esac fi -__cron letsencrypt-certbot \ - --user root \ - --command "$certbot_fullpath renew -q $renew_hook" \ - --hour 0 \ - --minute 47 +if [ -f "${__object}/parameter/automatic-renewal" ]; then + renew_hook_param="${__object}/parameter/renew-hook" + renew_hook="" + if [ -f "${renew_hook_param}" ]; then + while read hook; do + renew_hook="${renew_hook} --renew-hook \"${hook}\"" + done < "${renew_hook_param}" + fi + + __cron letsencrypt-certbot \ + --user root \ + --command "${certbot_fullpath} renew -q ${renew_hook}" \ + --hour 0 \ + --minute 47 +fi diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/boolean b/cdist/conf/type/__letsencrypt_cert/parameter/boolean new file mode 100644 index 00000000..d5b8be99 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/boolean @@ -0,0 +1,2 @@ +automatic-renewal +staging diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/default/state b/cdist/conf/type/__letsencrypt_cert/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/default/webroot b/cdist/conf/type/__letsencrypt_cert/parameter/default/webroot new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional b/cdist/conf/type/__letsencrypt_cert/parameter/optional new file mode 100644 index 00000000..0a63b11e --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional @@ -0,0 +1,2 @@ +state +webroot diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/required b/cdist/conf/type/__letsencrypt_cert/parameter/required index 45fe4ea6..bfe77226 100644 --- a/cdist/conf/type/__letsencrypt_cert/parameter/required +++ b/cdist/conf/type/__letsencrypt_cert/parameter/required @@ -1,2 +1 @@ admin-email -webroot diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/required_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/required_multiple new file mode 100644 index 00000000..d23ab7ab --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/required_multiple @@ -0,0 +1 @@ +domain From 72bb485db7964a37e2f3c93dc396f10b813ca1db Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 11 May 2018 18:29:43 +0200 Subject: [PATCH 045/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3bdf446b..64d2489e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) * New type: __docker_config (Ľubomír Kučera) * New type: __docker_secret (Ľubomír Kučera) + * Type __letsencrypt_cert: Rewritten; WARN: breaks backward compatibility (Ľubomír Kučera) 4.8.4: 2018-04-20 * Documentation, type manpages: Fix spelling (Dmitry Bogatov) From 3cf417fa2bb8152fd0bcef757ce29d3f24649d9c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 15 May 2018 12:12:01 +0200 Subject: [PATCH 046/178] Fix NameError: name 'cdist_object' is not defined, #662 --- cdist/core/explorer.py | 29 +++++++++++++---------------- docs/changelog | 1 + 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 072ca692..874d36f2 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -214,22 +214,19 @@ class Explorer(object): def transfer_type_explorers(self, cdist_type): """Transfer the type explorers for the given type to the remote side.""" - try: - if cdist_type.explorers: - if cdist_type.name in self._type_explorers_transferred: - self.log.trace(("Skipping retransfer of type explorers " - "for: %s"), cdist_type) - else: - source = os.path.join(self.local.type_path, - cdist_type.explorer_path) - destination = os.path.join(self.remote.type_path, - cdist_type.explorer_path) - self.remote.mkdir(destination) - self.remote.transfer(source, destination) - self.remote.run(["chmod", "0700", "%s/*" % (destination)]) - self._type_explorers_transferred.append(cdist_type.name) - except cdist.Error as e: - raise cdist.CdistObjectError(cdist_object, e) + if cdist_type.explorers: + if cdist_type.name in self._type_explorers_transferred: + self.log.trace(("Skipping retransfer of type explorers " + "for: %s"), cdist_type) + else: + source = os.path.join(self.local.type_path, + cdist_type.explorer_path) + destination = os.path.join(self.remote.type_path, + cdist_type.explorer_path) + self.remote.mkdir(destination) + self.remote.transfer(source, destination) + self.remote.run(["chmod", "0700", "%s/*" % (destination)]) + self._type_explorers_transferred.append(cdist_type.name) def transfer_object_parameters(self, cdist_object): """Transfer the parameters for the given object to the remote side.""" diff --git a/docs/changelog b/docs/changelog index 64d2489e..1e3787fd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * New type: __docker_config (Ľubomír Kučera) * New type: __docker_secret (Ľubomír Kučera) * Type __letsencrypt_cert: Rewritten; WARN: breaks backward compatibility (Ľubomír Kučera) + * Core: Fix NameError: name 'cdist_object' is not defined (Darko Poljak) 4.8.4: 2018-04-20 * Documentation, type manpages: Fix spelling (Dmitry Bogatov) From 6946dc877a6e25980f6d9368cee77bcccfbc68ef Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 May 2018 13:33:58 +0200 Subject: [PATCH 047/178] pep8 --- cdist/core/explorer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 874d36f2..8c18a19f 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -220,9 +220,9 @@ class Explorer(object): "for: %s"), cdist_type) else: source = os.path.join(self.local.type_path, - cdist_type.explorer_path) + cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, - cdist_type.explorer_path) + cdist_type.explorer_path) self.remote.mkdir(destination) self.remote.transfer(source, destination) self.remote.run(["chmod", "0700", "%s/*" % (destination)]) From e6b6925908c3daf01c3b16c1714a5b9e9184c926 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 May 2018 16:11:05 +0200 Subject: [PATCH 048/178] Release 4.9.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 1e3787fd..da23589c 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.9.0: 2018-05-17 * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) * New type: __docker_config (Ľubomír Kučera) * New type: __docker_secret (Ľubomír Kučera) From a3968f831306534c32aa3873e64c2e5a7fdfc811 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 18 May 2018 01:25:35 +0200 Subject: [PATCH 049/178] rewrite __line type for --before and --after support Signed-off-by: Steven Armstrong --- cdist/conf/type/__line/explorer/state | 72 +++++++-- cdist/conf/type/__line/gencode-remote | 148 ++++++++++-------- cdist/conf/type/__line/man.rst | 100 +++++++----- .../conf/type/__line/parameter/default/state | 1 + cdist/conf/type/__line/parameter/optional | 6 +- 5 files changed, 211 insertions(+), 116 deletions(-) create mode 100644 cdist/conf/type/__line/parameter/default/state diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index 08056c86..1f81b540 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -17,26 +17,64 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -file="/$__object_id" -[ -f "$__object/parameter/file" ] && file=$(cat "$__object/parameter/file") +if [ -f "$__object/parameter/before" ]; then + position="before" +elif [ -f "$__object/parameter/after" ]; then + position="after" +fi if [ -f "$__object/parameter/regex" ]; then - regex=$(cat "$__object/parameter/regex") - greparg="" + needle="regex" else - if [ ! -f "$__object/parameter/line" ]; then - echo "Parameter line and regex missing - cannot explore" >&2 - exit 1 - fi - regex="$(cat "$__object/parameter/line")" - greparg="-F -x" + needle="line" fi -# Allow missing file - thus 2>/dev/null -if grep -q $greparg -- "$regex" "$file" 2>/dev/null; then - echo present +if [ -f "$__object/parameter/file" ]; then + file="$(cat "$__object/parameter/file")" else - echo absent + file="/$__object_id" fi + +awk -v position="$position" -v needle="$needle" ' +BEGIN { + getline anchor < (ENVIRON["__object"] "/parameter/" position) + getline pattern < (ENVIRON["__object"] "/parameter/" needle) + state = "absent" +} +{ + if (position == "after") { + if (match($0, anchor)) { + getline + if (match($0, pattern)) { + state = "present" + } + else { + state = "wrongposition" + } + exit 0 + } + } + else if (position == "before") { + if (match($0, pattern)) { + getline + if (match($0, anchor)) { + state = "present" + } + else { + state = "wrongposition" + } + exit 0 + } + } + else { + if (match($0, pattern)) { + state = "present" + exit 0 + } + } +} +END { + print state +} +' "$file" diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 4a75b4c5..7951ea49 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # -# 2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -18,76 +17,101 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -file="/$__object_id" -regex="" -state_should="present" -[ -f "$__object/parameter/file" ] && file=$(cat "$__object/parameter/file") -[ -f "$__object/parameter/regex" ] && regex=$(cat "$__object/parameter/regex") -[ -f "$__object/parameter/state" ] && state_should=$(cat "$__object/parameter/state") -[ -f "$__object/parameter/line" ] && line=$(cat "$__object/parameter/line") +if [ -f "$__object/parameter/before" -a -f "$__object/parameter/after" ]; then + echo "Use either --before OR --after but not both." >&2 + exit 1 +fi +state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" -[ "$state_should" = "$state_is" ] && exit 0 +if [ "$state_should" = "$state_is" ]; then + # nothing to do + exit 0 +fi +if [ -f "$__object/parameter/before" ]; then + position="before" +elif [ -f "$__object/parameter/after" ]; then + position="after" +else + # By default we append to the end of the file. + position="end" +fi + +if [ -f "$__object/parameter/regex" ]; then + needle="regex" +else + needle="line" +fi + +if [ -f "$__object/parameter/file" ]; then + file="$(cat "$__object/parameter/file")" +else + file="/$__object_id" +fi + +add=0 +remove=0 case "$state_should" in - present) - if [ ! "$line" ]; then - echo "Required parameter \"line\" is missing" >&2 - exit 1 - fi + present) + if [ "$state_is" = "wrongposition" ]; then + echo updated >> "$__messages_out" + remove=1 + else + echo added >> "$__messages_out" + fi + add=1 + ;; + absent) + echo removed >> "$__messages_out" + remove=1 + ;; +esac - #echo "echo \"$line\" >> $file" - #line_sanitised=$(cat "$__object/parameter/line" | sed 's/"/\"/g') - # Idea: replace ' in the string: - # '"'"' - # |------> ': end the string - # |-|---> "'": create ' in the output string - # |--> ': continue the string - # - # Replace all \ so \t and other combinations are not interpreted - # - - - # line_sanitised=$(cat "$__object/parameter/line" | sed -e "s/'/'\"'\"'/g" -e 's/\\/\\\\/g') - # The one above does not work: - # --line "PS1='[\t] \[\033[1m\]\h\[\033[0m\]:\w\\$ '" - # becomes - # PS1='[\\t] \\[\\033[1m\\]\\h\\[\\033[0m\\]:\\w\\$ ' - - # Only replace ' with '"'"' and keep \ as they are - line_sanitised=$(cat "$__object/parameter/line" | sed -e "s/'/'\"'\"'/g") - printf '%s' "printf '%s\n' '$line_sanitised' >> $file" - echo "added" >> "$__messages_out" - - ;; - absent) - if [ "$regex" -a "$line" ]; then - echo "Mutally exclusive parameters regex and line given for state absent" >&2 - exit 1 - fi - - greparg="" - if [ "$line" ]; then - regex="$line" - greparg="-F -x" - fi - - cat << eof +cat << DONE tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" fi -grep -v $greparg "$regex" '$file' > \$tmpfile || true + +awk -v position="$position" -v needle="$needle" -v remove=$remove -v add=$add ' +BEGIN { + line_file = ENVIRON["__object"] "/parameter/line" + getline line < line_file + # Need to close line file as it may be re-read as pattern below. + close(line_file) + getline pattern < (ENVIRON["__object"] "/parameter/" needle) + getline anchor < (ENVIRON["__object"] "/parameter/" position) +} +{ + if (remove) { + if (match(\$0, pattern)) { + # skip over this line -> remove it + next + } + } + if (add) { + if (anchor && match(\$0, anchor)) { + if (position == "before") { + print line + print + } else if (position == "after") { + print + print line + } + next + } + } + print +} +END { + if (add && position == "end") { + print line + } +} +' "$file" > "\$tmpfile" mv -f "\$tmpfile" "$file" -eof - echo "removed" >> "$__messages_out" - ;; - *) - echo "Unknown state: $state_should" >&2 - exit 1 - ;; -esac +DONE diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index b63ea2b3..d651985e 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -13,72 +13,102 @@ This cdist type allows you to add lines and remove lines from files. REQUIRED PARAMETERS ------------------- +None. + OPTIONAL PARAMETERS ------------------- -state - 'present' or 'absent', defaults to 'present' +after + Insert the given line after this pattern. -line - Specifies the line which should be absent or present - - Must be present, if state is present. - Must not be combined with regex, if state is absent. - -regex - If state is present, search for this pattern and add - given line, if the given regular expression does not match. - - In case of absent, ensure all lines matching the - regular expression are absent. - - The regular expression is interpreted by grep. - - Must not be combined with line, if state is absent. +before + Insert the given line before this pattern. file If supplied, use this as the destination file. Otherwise the object_id is used. +line + Specifies the line which should be absent or present. + + Must be present, if state is 'present'. + Ignored if regex is given and state is 'absent'. + +regex + If state is 'present', search for this pattern and if it matches add + the given line. + + If state is 'absent', ensure all lines matching the regular expression + are absent. + + The regular expression is interpreted by awk's match function. + +state + 'present' or 'absent', defaults to 'present' + + + +BOOLEAN PARAMETERS +------------------ +None. + + MESSAGES -------- added - The line was added. + The line was added. + +updated + The line or its position was changed. removed - The line was removed. + The line was removed. + EXAMPLES -------- .. code-block:: sh - # Manage the DAEMONS line in rc.conf - __line daemons --file /etc/rc.conf --line 'DAEMONS=(hwclock !network sshd crond postfix)' + # Manage a hosts entry for www.example.com. + __line /etc/hosts \ + --line '127.0.0.2 www.example.com' - # Ensure the home mount is present in /etc/fstab - explicitly make it present - __line home-fstab \ - --file /etc/fstab \ - --line 'filer.fs:/vol/home /home nfs defaults 0 0' \ - --state present + # Manage another hosts entry for test.example.com. + __line hosts:test.example.com \ + --file /etc/hosts \ + --line '127.0.0.3 test.example.com' - # Removes the line specifiend in "include_www" from the file "lighttpd.conf" - __line legacy_timezone --file /etc/rc.conf --regex 'TIMEZONE=.*' --state absent + # Remove the line starting with TIMEZONE from the /etc/rc.conf file. + __line legacy_timezone \ + --file /etc/rc.conf \ + --regex 'TIMEZONE=.*' \ + --state absent + + # Insert a line before another one. + __line password-auth-local:classify \ + --file /etc/pam.d/password-auth-local \ + --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \ + --before '^session[[:space:]]+include[[:space:]]+password-auth-ac$' + + # Insert a line after another one. + __line password-auth-local:classify \ + --file /etc/pam.d/password-auth-local \ + --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \ + --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$' SEE ALSO -------- -:strong:`grep`\ (1) +:strong:`cdist-type`\ (7) AUTHORS ------- -Nico Schottelius +Steven Armstrong COPYING ------- -Copyright \(C) 2012-2013 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2018 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__line/parameter/default/state b/cdist/conf/type/__line/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__line/parameter/optional b/cdist/conf/type/__line/parameter/optional index 604a203e..f89a2115 100644 --- a/cdist/conf/type/__line/parameter/optional +++ b/cdist/conf/type/__line/parameter/optional @@ -1,4 +1,6 @@ -state -regex +after +before file line +regex +state From fb26894cbd45268b62eb40d4ed285dd557ea4f6c Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 18 May 2018 16:57:34 +0200 Subject: [PATCH 050/178] when searching treat line as string, and regex as regexp Signed-off-by: Steven Armstrong --- cdist/conf/type/__line/explorer/state | 13 ++++++++++--- cdist/conf/type/__line/gencode-remote | 9 ++++++++- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index 1f81b540..d27bca18 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -37,6 +37,13 @@ else fi awk -v position="$position" -v needle="$needle" ' +function _find(_text, _pattern) { + if (needle == "regex") { + return match(_text, _pattern) + } else { + return index(_text, _pattern) + } +} BEGIN { getline anchor < (ENVIRON["__object"] "/parameter/" position) getline pattern < (ENVIRON["__object"] "/parameter/" needle) @@ -46,7 +53,7 @@ BEGIN { if (position == "after") { if (match($0, anchor)) { getline - if (match($0, pattern)) { + if (_find($0, pattern)) { state = "present" } else { @@ -56,7 +63,7 @@ BEGIN { } } else if (position == "before") { - if (match($0, pattern)) { + if (_find($0, pattern)) { getline if (match($0, anchor)) { state = "present" @@ -68,7 +75,7 @@ BEGIN { } } else { - if (match($0, pattern)) { + if (_find($0, pattern)) { state = "present" exit 0 } diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 7951ea49..996029f5 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -78,6 +78,13 @@ if [ -f "$file" ]; then fi awk -v position="$position" -v needle="$needle" -v remove=$remove -v add=$add ' +function _find(_text, _pattern) { + if (needle == "regex") { + return match(_text, _pattern) + } else { + return index(_text, _pattern) + } +} BEGIN { line_file = ENVIRON["__object"] "/parameter/line" getline line < line_file @@ -88,7 +95,7 @@ BEGIN { } { if (remove) { - if (match(\$0, pattern)) { + if (_find(\$0, pattern)) { # skip over this line -> remove it next } From e7e8f1428aafbffc8c391527ec5c84c8710f4082 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Sat, 19 May 2018 17:42:25 +0200 Subject: [PATCH 051/178] Add new type __install_coreos --- .../conf/type/__install_coreos/gencode-remote | 19 +++++++ cdist/conf/type/__install_coreos/install | 0 cdist/conf/type/__install_coreos/man.rst | 50 +++++++++++++++++++ .../parameter/default/ignition | 0 .../type/__install_coreos/parameter/optional | 1 + .../type/__install_coreos/parameter/required | 1 + cdist/conf/type/__install_coreos/singleton | 0 7 files changed, 71 insertions(+) create mode 100755 cdist/conf/type/__install_coreos/gencode-remote create mode 100644 cdist/conf/type/__install_coreos/install create mode 100644 cdist/conf/type/__install_coreos/man.rst create mode 100644 cdist/conf/type/__install_coreos/parameter/default/ignition create mode 100644 cdist/conf/type/__install_coreos/parameter/optional create mode 100644 cdist/conf/type/__install_coreos/parameter/required create mode 100644 cdist/conf/type/__install_coreos/singleton diff --git a/cdist/conf/type/__install_coreos/gencode-remote b/cdist/conf/type/__install_coreos/gencode-remote new file mode 100755 index 00000000..fbe86479 --- /dev/null +++ b/cdist/conf/type/__install_coreos/gencode-remote @@ -0,0 +1,19 @@ +#!/bin/sh -e + +device=$(cat "${__object:?}/parameter/device") +ignition=$(cat "${__object}/parameter/ignition") + +cat < "\${ignition_file}" << eof +$(base64 "${ignition}") +eof + +coreos-install -d "${device}" \ + \$(if [ -s "\${ignition_file}" ]; then + printf -- "-i \${ignition_file}\n" + fi) + +rm "\${ignition_file}" +EOF diff --git a/cdist/conf/type/__install_coreos/install b/cdist/conf/type/__install_coreos/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_coreos/man.rst b/cdist/conf/type/__install_coreos/man.rst new file mode 100644 index 00000000..314f9f2a --- /dev/null +++ b/cdist/conf/type/__install_coreos/man.rst @@ -0,0 +1,50 @@ +cdist-type__install_coreos(7) +============================= + +NAME +---- + +cdist-type__install_coreos - Install CoreOS + +DESCRIPTION +----------- + +This type installs CoreOS to a given device using coreos-install_, which is +present in CoreOS ISO by default. + +.. _coreos-install: https://raw.githubusercontent.com/coreos/init/master/bin/coreos-install + +REQUIRED PARAMETERS +------------------- + +device + A device CoreOS will be installed to. + +OPTIONAL PARAMETERS +------------------- + +ignition + Path to ignition config. + +EXAMPLES +-------- + +.. code-block:: sh + + __install_coreos \ + --device /dev/sda \ + --ignition ignition.json + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_coreos/parameter/default/ignition b/cdist/conf/type/__install_coreos/parameter/default/ignition new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_coreos/parameter/optional b/cdist/conf/type/__install_coreos/parameter/optional new file mode 100644 index 00000000..df284caa --- /dev/null +++ b/cdist/conf/type/__install_coreos/parameter/optional @@ -0,0 +1 @@ +ignition diff --git a/cdist/conf/type/__install_coreos/parameter/required b/cdist/conf/type/__install_coreos/parameter/required new file mode 100644 index 00000000..f89ee6a8 --- /dev/null +++ b/cdist/conf/type/__install_coreos/parameter/required @@ -0,0 +1 @@ +device diff --git a/cdist/conf/type/__install_coreos/singleton b/cdist/conf/type/__install_coreos/singleton new file mode 100644 index 00000000..e69de29b From ab696a8cb86800576db4c0d24c33338778041059 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 25 May 2018 18:01:26 +0200 Subject: [PATCH 052/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index da23589c..7a811271 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __install_coreos (Ľubomír Kučera) + 4.9.0: 2018-05-17 * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) * New type: __docker_config (Ľubomír Kučera) From b3a4152e00f3a86fbc741dd02ba839e956216490 Mon Sep 17 00:00:00 2001 From: Alexander Dinu Date: Sun, 27 May 2018 17:20:55 +0200 Subject: [PATCH 053/178] Fix __package_yum explorer Why: In case when name contains package name with exact version specified (e.g. rpm-build-4.11.3) ``` rpm -q --whatprovides "$pkg_name" ``` will tell you that no package could provide you with 'rpm-build-4.11.3', because it's not virtual or file-provide, but exact package name. This will lead to the installation of the package which was already installed. It slows down manifest execution a lot. My change will keep previous behaviour which relies on --whatprovides and will fix wrong behaviour when argument is full package name with version. --- cdist/conf/type/__package_yum/explorer/pkg_version | 2 +- cdist/conf/type/__package_yum/gencode-remote | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__package_yum/explorer/pkg_version b/cdist/conf/type/__package_yum/explorer/pkg_version index fb3b7753..b81b0fe9 100755 --- a/cdist/conf/type/__package_yum/explorer/pkg_version +++ b/cdist/conf/type/__package_yum/explorer/pkg_version @@ -27,4 +27,4 @@ else name="$__object_id" fi -rpm -q --whatprovides "$name" 2>/dev/null || true +rpm -q "$name" 2>/dev/null || rpm -q --whatprovides "$name" 2>/dev/null || true diff --git a/cdist/conf/type/__package_yum/gencode-remote b/cdist/conf/type/__package_yum/gencode-remote index e9b48ee8..e1323dea 100755 --- a/cdist/conf/type/__package_yum/gencode-remote +++ b/cdist/conf/type/__package_yum/gencode-remote @@ -43,10 +43,15 @@ else opts="--assumeyes --quiet" fi -not_installed="^no package provides" +not_provided="^no package provides" +not_installed='is not installed$' -if grep -q "$not_installed" "$__object/explorer/pkg_version"; then - state_is="absent" +if grep -q "$not_provided" "$__object/explorer/pkg_version"; then + if grep -q "$not_installed" "$__object/explorer/pkg_version"; then + state_is="absent" + else + state_is="present" + fi else state_is="present" fi From 709c216096fd6a9249d1a817e6e33864dd64a2b7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 27 May 2018 19:07:57 +0200 Subject: [PATCH 054/178] [__consul_agent] Add LSB header so that innserv doe not fail --- .../conf/type/__consul_agent/files/consul.sysv-debian | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cdist/conf/type/__consul_agent/files/consul.sysv-debian b/cdist/conf/type/__consul_agent/files/consul.sysv-debian index a75c555d..098328b3 100644 --- a/cdist/conf/type/__consul_agent/files/consul.sysv-debian +++ b/cdist/conf/type/__consul_agent/files/consul.sysv-debian @@ -18,6 +18,16 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +### BEGIN INIT INFO +# Provides: consul +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $local_fs +# Should-Start: +# Should-Stop: +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +# Short-Description: libvirt management daemon +### END INIT INFO if [ -f "/etc/default/consul" ]; then . /etc/default/consul From ab62d5871545cb6df3103d7d8f7e4432ebd319f6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 27 May 2018 19:09:24 +0200 Subject: [PATCH 055/178] ++ changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 7a811271..bf88e047 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * New type: __install_coreos (Ľubomír Kučera) + * Type __consul_agent: Add LSB init header (Nico Schottelius) 4.9.0: 2018-05-17 * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) From 81adabd1ab286d22e15fa2ac8c89bc3f8e72222a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 May 2018 08:04:41 +0200 Subject: [PATCH 056/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bf88e047..99d9bb81 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * New type: __install_coreos (Ľubomír Kučera) * Type __consul_agent: Add LSB init header (Nico Schottelius) + * Type __package_yum: Fix explorer when name contains package name with exact version specified (Aleksandr Dinu) 4.9.0: 2018-05-17 * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) From efebe153868e0fd6a51a89387097dc847a7ae14e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 27 May 2018 15:32:44 +0200 Subject: [PATCH 057/178] Fix __letsencrypt_cert type: use object id as domain if domain param is not specified --- .../type/__letsencrypt_cert/gencode-remote | 8 +++++- cdist/conf/type/__letsencrypt_cert/man.rst | 26 ++++++++++++++----- .../parameter/optional_multiple | 1 + .../parameter/required_multiple | 1 - docs/changelog | 1 + 5 files changed, 28 insertions(+), 9 deletions(-) delete mode 100644 cdist/conf/type/__letsencrypt_cert/parameter/required_multiple diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote index 4bd05eff..788da74c 100755 --- a/cdist/conf/type/__letsencrypt_cert/gencode-remote +++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote @@ -15,7 +15,13 @@ case "${state}" in echo remove >> "${__messages_out:?}" ;; present) - requested_domains="${__object}/parameter/domain" + domain_param_file="${__object}/parameter/domain" + requested_domains=$(mktemp domain.cdist.XXXXXXXXXX) + if [ -f "${domain_param_file}" ]; then + cp "${domain_param_file}" "${requested_domains}" + else + echo "$__object_id" >> "${requested_domains}" + fi staging=no if [ -f "${__object}/parameter/staging" ]; then diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst index 16b23fea..c4ffc6bc 100644 --- a/cdist/conf/type/__letsencrypt_cert/man.rst +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -14,15 +14,13 @@ Automatically obtain a Let's Encrypt SSL certificate using Certbot. REQUIRED PARAMETERS ------------------- +object id + A cert name. If domain parameter is not specified then it is used + as a domain to be included in the certificate. + admin-email Where to send Let's Encrypt emails like "certificate needs renewal". -REQUIRED MULTIPLE PARAMETERS ----------------------------- - -domain - A domain to be included in the certificate. - OPTIONAL PARAMETERS ------------------- @@ -44,6 +42,10 @@ OPTIONAL MULTIPLE PARAMETERS renew-hook Renew hook command directly passed to Certbot in cron job. +domain + Domains to be included in the certificate. When specified then object id + is not used as a domain. + BOOLEAN PARAMETERS ------------------ @@ -70,6 +72,17 @@ EXAMPLES .. code-block:: sh + # use object id as domain + __letsencrypt_cert example.com \ + --admin-email root@example.com \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --webroot /data/letsencrypt/root + +.. code-block:: sh + + # domain parameter is specified so object id is not used as domain + # and example.com needs to be included again with domain parameter __letsencrypt_cert example.com \ --admin-email root@example.com \ --automatic-renewal \ @@ -79,7 +92,6 @@ EXAMPLES --renew-hook "service nginx reload" \ --webroot /data/letsencrypt/root - AUTHORS ------- diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple index 3384c74f..0e866d45 100644 --- a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple +++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple @@ -1 +1,2 @@ +domain renew-hook diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/required_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/required_multiple deleted file mode 100644 index d23ab7ab..00000000 --- a/cdist/conf/type/__letsencrypt_cert/parameter/required_multiple +++ /dev/null @@ -1 +0,0 @@ -domain diff --git a/docs/changelog b/docs/changelog index 99d9bb81..250baae5 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * New type: __install_coreos (Ľubomír Kučera) * Type __consul_agent: Add LSB init header (Nico Schottelius) * Type __package_yum: Fix explorer when name contains package name with exact version specified (Aleksandr Dinu) + * Type __letsencrypt_cert: Use object id as domain if domain param is not specified (Darko Poljak) 4.9.0: 2018-05-17 * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) From 5ec99ceda2a3d6a3c7478d46f9f941e89be3ca0d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 30 May 2018 19:44:54 +0200 Subject: [PATCH 058/178] Release 4.9.1 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 250baae5..38411e14 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.9.1: 2018-05-30 * New type: __install_coreos (Ľubomír Kučera) * Type __consul_agent: Add LSB init header (Nico Schottelius) * Type __package_yum: Fix explorer when name contains package name with exact version specified (Aleksandr Dinu) From 4516ee0baabef2c3a612d624a60fce27fd43ced0 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 1 Jun 2018 15:27:40 +0200 Subject: [PATCH 059/178] position can not be empty Signed-off-by: Steven Armstrong --- cdist/conf/type/__line/explorer/state | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index d27bca18..afdf3502 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -22,6 +22,9 @@ if [ -f "$__object/parameter/before" ]; then position="before" elif [ -f "$__object/parameter/after" ]; then position="after" +else + # By default we append to the end of the file. + position="end" fi if [ -f "$__object/parameter/regex" ]; then From 90adefe2e455e7ab8cef2629145ec0ad7136fbc4 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 7 Jun 2018 15:07:00 +0300 Subject: [PATCH 060/178] add __acl: Basic wrapper around setfacl --- cdist/conf/type/__acl/explorer/acl_is | 23 ++++++ cdist/conf/type/__acl/gencode-remote | 81 +++++++++++++++++++ cdist/conf/type/__acl/man.rst | 62 ++++++++++++++ cdist/conf/type/__acl/parameter/boolean | 3 + .../type/__acl/parameter/optional_multiple | 2 + 5 files changed, 171 insertions(+) create mode 100755 cdist/conf/type/__acl/explorer/acl_is create mode 100755 cdist/conf/type/__acl/gencode-remote create mode 100644 cdist/conf/type/__acl/man.rst create mode 100644 cdist/conf/type/__acl/parameter/boolean create mode 100644 cdist/conf/type/__acl/parameter/optional_multiple diff --git a/cdist/conf/type/__acl/explorer/acl_is b/cdist/conf/type/__acl/explorer/acl_is new file mode 100755 index 00000000..4dc98c51 --- /dev/null +++ b/cdist/conf/type/__acl/explorer/acl_is @@ -0,0 +1,23 @@ +#!/bin/sh -e +# +# 2018 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if [ -e "/$__object_id" ] +then getfacl "/$__object_id" | grep -E '^((default:|)(user|group)):[a-z]' || true +fi diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote new file mode 100755 index 00000000..a9d14102 --- /dev/null +++ b/cdist/conf/type/__acl/gencode-remote @@ -0,0 +1,81 @@ +#!/bin/sh -e +# +# 2018 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os="$( "$__explorer/os" )" + +acl_path="/$__object_id" + +acl_is="$( cat "$__object/explorer/acl_is" )" + +acl_should="$( for parameter in user group +do + if [ ! -f "$__object/parameter/$parameter" ] + then continue + fi + while read -r l + do + echo "$parameter:$l" + + if [ -f "$__object/parameter/default" ] + then echo "default:$parameter:$l" + fi + done < "$__object/parameter/$parameter" +done )" + +setfacl_exec='setfacl' + +if [ -f "$__object/parameter/recursive" ] +then + if echo "$os" | grep -E 'macosx|netbsd|freebsd|openbsd' + then + echo "$os setfacl do not support recursive operations" >&2 + else + setfacl_exec="$setfacl_exec -R" + fi +fi + +if [ -f "$__object/parameter/remove" ] +then + if echo "$os" | grep 'solaris' + then + # Solaris setfacl behaves differently. + # We will not support Solaris for now, because no way to test it. + # But adding support should be easy (use -s instead of -m on modify). + echo "$os setfacl do not support -x flag for ACL remove" >&2 + else + echo "$acl_is" | while read -r acl + do + if echo "$acl_should" | grep -Fq "$acl" + then continue + fi + + no_bits="$( echo "$acl" | sed -r 's/:[rwx-]+$//' )" + + echo "$setfacl_exec -x \"$no_bits\" \"$acl_path\"" + done + fi +fi + +for acl in $acl_should +do + if ! echo "$acl_is" | grep -Eq "^$acl" + then echo "$setfacl_exec -m \"$acl\" \"$acl_path\"" + fi +done diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst new file mode 100644 index 00000000..39db4d75 --- /dev/null +++ b/cdist/conf/type/__acl/man.rst @@ -0,0 +1,62 @@ +cdist-type__acl(7) +================== + +NAME +---- +cdist-type__acl - Basic wrapper around `setfacl` + + +DESCRIPTION +----------- +ACL must be defined as 3-symbol combination, using `r`, `w`, `x` and `-`. + +See setfacl(1) and acl(5) for more details. + + +OPTIONAL MULTIPLE PARAMETERS +---------------------------- +user + Add user ACL entry. + +group + Add group ACL entry. + + +BOOLEAN PARAMETERS +------------------ +recursive + Operate recursively (Linux only). + +default + Add default ACL entries. + +remove + Remove undefined ACL entries (Solaris not supported). + + +EXAMPLES +-------- + +.. code-block:: sh + + __acl /srv/project \ + --recursive \ + --default \ + --remove \ + --user alice:rwx \ + --user bob:r-x \ + --group project-group:rwx \ + --group some-other-group:r-x + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2018 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__acl/parameter/boolean b/cdist/conf/type/__acl/parameter/boolean new file mode 100644 index 00000000..8b96693f --- /dev/null +++ b/cdist/conf/type/__acl/parameter/boolean @@ -0,0 +1,3 @@ +recursive +default +remove diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple new file mode 100644 index 00000000..22f5a52c --- /dev/null +++ b/cdist/conf/type/__acl/parameter/optional_multiple @@ -0,0 +1,2 @@ +user +group From d7a1645e4ed3f9bca0d687c51feb209b9dbabb49 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 11 Jun 2018 11:21:07 +0300 Subject: [PATCH 061/178] __acl: fix os explorer --- cdist/conf/type/__acl/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index a9d14102..a59d49e0 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -18,7 +18,7 @@ # along with cdist. If not, see . # -os="$( "$__explorer/os" )" +os="$( cat "$__global/explorer/os" )" acl_path="/$__object_id" From b9a48c931689afe009075fc3ddc0b9ab7fb34d10 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 13 Jun 2018 06:20:02 +0200 Subject: [PATCH 062/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 38411e14..241f4933 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __acl (Ander Punnar) + 4.9.1: 2018-05-30 * New type: __install_coreos (Ľubomír Kučera) * Type __consul_agent: Add LSB init header (Nico Schottelius) From 5aa8dac80a93b4158a713a7837f460042ecd11fe Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 13 Jun 2018 07:53:52 +0200 Subject: [PATCH 063/178] Disable config parser interpolation --- cdist/configuration.py | 2 +- cdist/test/configuration/__init__.py | 66 ++++++++++++------- .../fixtures/interpolation-test.cfg | 2 + docs/changelog | 1 + 4 files changed, 45 insertions(+), 26 deletions(-) create mode 100644 cdist/test/configuration/fixtures/interpolation-test.cfg diff --git a/cdist/configuration.py b/cdist/configuration.py index 848956aa..8df43f9f 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -382,7 +382,7 @@ class Configuration(metaclass=Singleton): return args def _read_config_file(self, files): - config_parser = configparser.ConfigParser() + config_parser = configparser.ConfigParser(interpolation=None) config_parser.read(files) d = dict() for section in config_parser.sections(): diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index e34c30b9..3319d320 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -31,6 +31,11 @@ import logging my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') +interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") + + +def newConfigParser(): + return configparser.ConfigParser(interpolation=None) class ConfigurationOptionsTestCase(test.CdistTestCase): @@ -141,7 +146,7 @@ class ConfigurationTestCase(test.CdistTestCase): def setUp(self): # Create test config file. - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -159,13 +164,13 @@ class ConfigurationTestCase(test.CdistTestCase): 'verbosity': 'INFO', 'archiving': 'none', } - config_custom = configparser.ConfigParser() + config_custom = newConfigParser() config_custom['GLOBAL'] = { 'parallel': '4', 'archiving': 'txz', } - config_custom2 = configparser.ConfigParser() + config_custom2 = newConfigParser() config_custom2['GLOBAL'] = { 'parallel': '16', 'archiving': 'tbz2', @@ -405,7 +410,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -463,7 +468,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -486,7 +491,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(global_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'on', 'local_shell': '/usr/bin/sh', @@ -544,7 +549,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -611,7 +616,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -634,7 +639,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(global_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'on', 'local_shell': '/usr/bin/sh', @@ -709,7 +714,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -732,7 +737,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(global_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'on', 'local_shell': '/usr/bin/sh', @@ -800,7 +805,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -823,7 +828,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(global_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'on', 'local_shell': '/usr/bin/sh', @@ -840,7 +845,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(local_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'conf_dir': '/opt/conf/cdist', 'remote_copy': 'scpcustom', @@ -899,7 +904,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -922,7 +927,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(global_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'on', 'local_shell': '/usr/bin/sh', @@ -939,7 +944,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(local_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'conf_dir': '/opt/conf/cdist', 'remote_copy': 'scpcustom', @@ -998,7 +1003,7 @@ class ConfigurationTestCase(test.CdistTestCase): } args = argparse.Namespace() - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'off', 'local_shell': '/bin/sh', @@ -1021,7 +1026,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(global_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'beta': 'on', 'local_shell': '/usr/bin/sh', @@ -1038,7 +1043,7 @@ class ConfigurationTestCase(test.CdistTestCase): with open(local_config_file, 'w') as f: config.write(f) - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'conf_dir': '/opt/conf/cdist', 'remote_copy': 'scpcustom', @@ -1107,7 +1112,7 @@ class ConfigurationTestCase(test.CdistTestCase): self.assertEqual(dargs, expected_args) def test_configuration_empty_value_in_file(self): - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'inventory_dir': '', 'conf_dir': '', @@ -1169,7 +1174,7 @@ class ConfigurationTestCase(test.CdistTestCase): config_files=()) def test_configuration_disable_saving_output_streams1(self): - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'save_output_streams': 'True', } @@ -1197,7 +1202,7 @@ class ConfigurationTestCase(test.CdistTestCase): self.assertEqual(configuration.config, expected_config_dict) def test_configuration_disable_saving_output_streams2(self): - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'save_output_streams': 'False', } @@ -1225,7 +1230,7 @@ class ConfigurationTestCase(test.CdistTestCase): self.assertEqual(configuration.config, expected_config_dict) def test_configuration_disable_saving_output_streams3(self): - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'save_output_streams': 'False', } @@ -1253,7 +1258,7 @@ class ConfigurationTestCase(test.CdistTestCase): self.assertEqual(configuration.config, expected_config_dict) def test_configuration_disable_saving_output_streams4(self): - config = configparser.ConfigParser() + config = newConfigParser() config['GLOBAL'] = { 'save_output_streams': 'True', } @@ -1280,6 +1285,17 @@ class ConfigurationTestCase(test.CdistTestCase): config_files=config_files) self.assertEqual(configuration.config, expected_config_dict) + def test_read_config_file_with_interpolation(self): + try: + config = cc.Configuration(None, env={}, config_files=()) + d = config._read_config_file(interpolation_config_file) + val = d['GLOBAL']['cache_path_pattern'] + self.assertIsNotNone(val) + self.assertEqual(val, '%N') + except configparser.InterpolationSyntaxError as e: + self.fail("Exception should not have been raised: {}".format( + e)) + if __name__ == "__main__": import unittest diff --git a/cdist/test/configuration/fixtures/interpolation-test.cfg b/cdist/test/configuration/fixtures/interpolation-test.cfg new file mode 100644 index 00000000..df723121 --- /dev/null +++ b/cdist/test/configuration/fixtures/interpolation-test.cfg @@ -0,0 +1,2 @@ +[GLOBAL] +cache_path_pattern = %N diff --git a/docs/changelog b/docs/changelog index 241f4933..6a1e1318 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * New type: __acl (Ander Punnar) + * Core: Disable config parser interpolation (Darko Poljak) 4.9.1: 2018-05-30 * New type: __install_coreos (Ľubomír Kučera) From 9d4c0331a9fd4772162c8ff98cdcae91040c94ac Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 15 Jun 2018 12:54:58 +0200 Subject: [PATCH 064/178] __sysctl: support sysctl.d --- cdist/conf/type/__sysctl/explorer/conf-path | 25 +++++++++++++++++++++ cdist/conf/type/__sysctl/manifest | 4 +++- 2 files changed, 28 insertions(+), 1 deletion(-) create mode 100755 cdist/conf/type/__sysctl/explorer/conf-path diff --git a/cdist/conf/type/__sysctl/explorer/conf-path b/cdist/conf/type/__sysctl/explorer/conf-path new file mode 100755 index 00000000..ba35c4c6 --- /dev/null +++ b/cdist/conf/type/__sysctl/explorer/conf-path @@ -0,0 +1,25 @@ +#!/bin/sh +# +# 2018 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if [ -d "/etc/sysctl.d" ]; then + echo "/etc/sysctl.d/99-Z-sysctl-cdist.conf"; +else + echo "/etc/sysctl.conf"; +fi diff --git a/cdist/conf/type/__sysctl/manifest b/cdist/conf/type/__sysctl/manifest index c903dbae..6e337ccb 100755 --- a/cdist/conf/type/__sysctl/manifest +++ b/cdist/conf/type/__sysctl/manifest @@ -32,8 +32,10 @@ case "$os" in ;; esac +conf_path=$(cat "$__object/explorer/conf-path") + __key_value "$__object_name" \ --key "$__object_id" \ - --file /etc/sysctl.conf \ + --file "${conf_path}" \ --value "$(cat "$__object/parameter/value")" \ --delimiter '=' From d49daca3dc9e3af09e42ea37dc38d561cf3d5743 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 16 Jun 2018 10:51:46 +0200 Subject: [PATCH 065/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 6a1e1318..509d0c44 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,8 @@ Changelog next: * New type: __acl (Ander Punnar) * Core: Disable config parser interpolation (Darko Poljak) + * Type __sysctl: Use sysctl.d location if exists (Darko Poljak) + * Type __line: Rewrite and support --before and --after (Steven Armstrong) 4.9.1: 2018-05-30 * New type: __install_coreos (Ľubomír Kučera) From ee64936ff9f07a2129a6c809db0512aafd2d70bc Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 17 Jun 2018 10:57:06 +0200 Subject: [PATCH 066/178] Release 4.10.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 509d0c44..0001b346 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.10.0: 2018-06-17 * New type: __acl (Ander Punnar) * Core: Disable config parser interpolation (Darko Poljak) * Type __sysctl: Use sysctl.d location if exists (Darko Poljak) From ec8f2f9488ef6dc7f6bc3ae282979555f544e82c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 20 Jun 2018 08:29:41 +0200 Subject: [PATCH 067/178] Fix temp file location and removal --- cdist/conf/type/__letsencrypt_cert/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote index 788da74c..375570a4 100755 --- a/cdist/conf/type/__letsencrypt_cert/gencode-remote +++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote @@ -16,7 +16,7 @@ case "${state}" in ;; present) domain_param_file="${__object}/parameter/domain" - requested_domains=$(mktemp domain.cdist.XXXXXXXXXX) + requested_domains=$(mktemp "${TMPDIR:-/tmp}/domain.cdist.XXXXXXXXXX") if [ -f "${domain_param_file}" ]; then cp "${domain_param_file}" "${requested_domains}" else @@ -66,6 +66,7 @@ case "${state}" in echo "--domain '${domain}' \\" done < "${requested_domains}") EOF + rm -f "${requested_domains}" if [ "${certificate_exists}" = "no" ]; then echo create >> "${__messages_out}" From 41186d66d406a3eb411e6fc1e8aa4b68ffbd49a7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 20 Jun 2018 09:23:12 +0200 Subject: [PATCH 068/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 0001b346..b81c4939 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) + 4.10.0: 2018-06-17 * New type: __acl (Ander Punnar) * Core: Disable config parser interpolation (Darko Poljak) From 4eaace7dd5c427150cf99b07893bff7c1d072f8c Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Wed, 20 Jun 2018 11:14:25 +0200 Subject: [PATCH 069/178] Handle missing file in __line explorer gracefully closes #675 --- cdist/conf/type/__line/explorer/state | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index afdf3502..2ef252c8 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -39,6 +39,11 @@ else file="/$__object_id" fi +if [ ! -f "$file" ]; then + echo "file_missing" + exit 0 +fi + awk -v position="$position" -v needle="$needle" ' function _find(_text, _pattern) { if (needle == "regex") { From 58b0e83655895f5d92c7c4c63ba4b79651703ee6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 20 Jun 2018 17:28:30 +0200 Subject: [PATCH 070/178] Add env vars usage idiom for writing types --- docs/src/cdist-type.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index a6587328..7c0dab8d 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -331,6 +331,19 @@ So when you generate a script with the following content, it will work: fi +Environment variable usage idiom +-------------------------------- +In type scripts you can support environment variables with default values if +environment variable is unset or null by using **${parameter:-[word]}** +parameter expansion. + +Example using mktemp in a portable way that supports TMPDIR environment variable. + +.. code-block:: sh + + tempfile=$(mktemp "${TMPDIR:-/tmp}/cdist.XXXXXXXXXX") + + Log level in types ------------------ cdist log level can be accessed from __cdist_log_level variable.One of: From 8350b6297ea2b03005d5b7beaaec64e02c8d4ff3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 20 Jun 2018 17:28:38 +0200 Subject: [PATCH 071/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index b81c4939..3794221f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) + * Type __line: Handle missing file in __line explorer gracefully (Jonas Weber) + * Documentation: Add env vars usage idiom for writing types (Darko Poljak) 4.10.0: 2018-06-17 * New type: __acl (Ander Punnar) From 7eae68c11da10d294e181f099dd2c58fffc81be6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 21 Jun 2018 08:33:48 +0200 Subject: [PATCH 072/178] Release 4.10.1 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3794221f..4453b6cb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) * Type __line: Handle missing file in __line explorer gracefully (Jonas Weber) * Documentation: Add env vars usage idiom for writing types (Darko Poljak) From 6c8014b407cf28ee54764ffc6182cb28a580dc66 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 22 Jun 2018 14:09:31 +0200 Subject: [PATCH 073/178] __letsencrypt_cert: add support for devuan ascii --- cdist/conf/type/__letsencrypt_cert/manifest | 11 +++++++++++ docs/changelog | 3 +++ 2 files changed, 14 insertions(+) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 94e9d225..56e3532c 100755 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -56,6 +56,17 @@ if [ -z "${certbot_fullpath}" ]; then # Seems to be a missing dependency on debian 8 __package python-ndg-httpsclient ;; + ascii*) + __apt_source ascii-backports \ + --uri http://auto.mirror.devuan.org/merged \ + --distribution ascii-backports \ + --component main + + require="__apt_source/ascii-backports" __package_apt python-certbot \ + --target-release ascii-backports + require="__apt_source/ascii-backports" __package_apt certbot \ + --target-release ascii-backports + ;; *) echo "Unsupported OS version: $os_version" >&2 exit 1 diff --git a/docs/changelog b/docs/changelog index 4453b6cb..921e0c26 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) + 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) * Type __line: Handle missing file in __line explorer gracefully (Jonas Weber) From 8d84834db6f2bf16474161cff358496d76923801 Mon Sep 17 00:00:00 2001 From: Adam Dej Date: Wed, 11 Jul 2018 15:53:23 +0200 Subject: [PATCH 074/178] __systemd_unit: manpage clarifications Clarify that this type only operates on units in /etc/systemd/system. Also, when state=present, it is not always true that the type is "installed, enabled and started" --- cdist/conf/type/__systemd_unit/man.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__systemd_unit/man.rst b/cdist/conf/type/__systemd_unit/man.rst index 88da6b30..5d68b253 100644 --- a/cdist/conf/type/__systemd_unit/man.rst +++ b/cdist/conf/type/__systemd_unit/man.rst @@ -9,9 +9,10 @@ cdist-type__systemd_unit - Install a systemd unit DESCRIPTION ----------- -This type can install, enable and start a systemd unit. This is particularly -useful on systems which take advantage of systemd heavily (e.g., CoreOS). For -more information about systemd units, see SYSTEMD.UNIT(5). +This type manages systemd units in ``/etc/systemd/system/``. It can install, +enable and start a systemd unit. This is particularly useful on systems which +take advantage of systemd heavily (e.g., CoreOS). For more information about +systemd units, see SYSTEMD.UNIT(5). REQUIRED PARAMETERS ------------------- @@ -37,7 +38,7 @@ state 'present' or 'absent', defaults to 'present' where: present - the unit is installed, enabled and started + the unit is installed absent the unit is stopped, disabled and uninstalled From 37b37f6e66d1d2d85b3eb2178787aec03221d2a6 Mon Sep 17 00:00:00 2001 From: Adam Dej Date: Wed, 11 Jul 2018 15:58:41 +0200 Subject: [PATCH 075/178] __systemd_unit: fix crash when transitioning to state=absent This type tried to disable an unit after it has removed it, which failed. Now the removal happens in gencode-remote, after the unit has been stopped and disabled. --- cdist/conf/type/__systemd_unit/gencode-remote | 2 ++ cdist/conf/type/__systemd_unit/manifest | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__systemd_unit/gencode-remote b/cdist/conf/type/__systemd_unit/gencode-remote index c608d9b3..9c64b6c4 100644 --- a/cdist/conf/type/__systemd_unit/gencode-remote +++ b/cdist/conf/type/__systemd_unit/gencode-remote @@ -25,6 +25,8 @@ current_enablement_state=$(cat "${__object}/explorer/enablement-state") if [ "${state}" = "absent" ]; then if [ ! -z "${current_enablement_state}" ]; then echo "systemctl --now disable ${name}" + echo "rm -f /etc/systemd/system/${name}" + echo "systemctl daemon-reload" fi exit 0 diff --git a/cdist/conf/type/__systemd_unit/manifest b/cdist/conf/type/__systemd_unit/manifest index 8b136605..7739c3d8 100644 --- a/cdist/conf/type/__systemd_unit/manifest +++ b/cdist/conf/type/__systemd_unit/manifest @@ -30,7 +30,9 @@ name="${__object_id}" source=$(cat "${__object}/parameter/source") state=$(cat "${__object}/parameter/state") -if [ -z "${source}" ] && [ "${state}" != "absent" ]; then +# The unit must be disabled before removing its unit file. The unit file is +# therefore removed by gencode-remote of this type, not here. +if [ -z "${source}" ] || [ "${state}" = "absent" ]; then exit 0 fi From 43982f821f6cae64a99673cf9541b5b0b0a940c1 Mon Sep 17 00:00:00 2001 From: Adam Dej Date: Wed, 11 Jul 2018 13:00:41 +0200 Subject: [PATCH 076/178] __systemd_unit: add support for masking units --- cdist/conf/type/__systemd_unit/gencode-remote | 17 ++++++++++++++--- cdist/conf/type/__systemd_unit/man.rst | 12 ++++++++---- cdist/conf/type/__systemd_unit/manifest | 12 +++++++++++- 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__systemd_unit/gencode-remote b/cdist/conf/type/__systemd_unit/gencode-remote index 9c64b6c4..60486c52 100644 --- a/cdist/conf/type/__systemd_unit/gencode-remote +++ b/cdist/conf/type/__systemd_unit/gencode-remote @@ -33,16 +33,24 @@ if [ "${state}" = "absent" ]; then fi unit_status=$(cat "${__object}/explorer/unit-status") +desired_enablement_state=$(cat "${__object}/parameter/enablement-state") + +if [ "${current_enablement_state}" = "masked" ] && \ + [ "${desired_enablement_state}" != "masked" ]; then + echo "systemctl unmask ${name}" +fi if [ -f "${__object}/parameter/restart" ]; then - if grep -q "^__file/etc/systemd/system/${name}" "${__messages_in}" || \ + if [ "${desired_enablement_state}" = "masked" ]; then + if [ "${unit_status}" = "active" ]; then + echo "systemctl stop ${name}" + fi + elif grep -q "^__file/etc/systemd/system/${name}" "${__messages_in}" || \ [ "${unit_status}" != "active" ]; then echo "systemctl restart ${name} || true" fi fi -desired_enablement_state=$(cat "${__object}/parameter/enablement-state") - if [ "${current_enablement_state}" = "${desired_enablement_state}" ]; then exit 0 fi @@ -58,6 +66,9 @@ case "${desired_enablement_state}" in disabled) echo "systemctl disable ${name}" ;; + masked) + echo "systemctl mask ${name}" + ;; *) echo "Unsupported unit status: ${desired_enablement_state}" >&2 exit 1 diff --git a/cdist/conf/type/__systemd_unit/man.rst b/cdist/conf/type/__systemd_unit/man.rst index 5d68b253..25a4e501 100644 --- a/cdist/conf/type/__systemd_unit/man.rst +++ b/cdist/conf/type/__systemd_unit/man.rst @@ -23,12 +23,14 @@ OPTIONAL PARAMETERS ------------------- enablement-state - 'enabled' or 'disabled', where: + 'enabled', 'disabled' or 'masked', where: enabled enables the unit disabled disables the unit + masked + masks the unit source Path to the config file. If source is '-' (dash), take what was written to @@ -38,15 +40,17 @@ state 'present' or 'absent', defaults to 'present' where: present - the unit is installed + the unit (or its mask) is installed absent - the unit is stopped, disabled and uninstalled + The unit is stopped, disabled and uninstalled. If the unit was masked, + the mask is removed. BOOLEAN PARAMETERS ------------------ restart - Restart the unit on unit file change or when the unit is inactive. + Start the unit if it was inactive. Restart the unit if the unit file + changed. Stop the unit if new ``enablement-state`` is ``masked``. MESSAGES -------- diff --git a/cdist/conf/type/__systemd_unit/manifest b/cdist/conf/type/__systemd_unit/manifest index 7739c3d8..688a00b1 100644 --- a/cdist/conf/type/__systemd_unit/manifest +++ b/cdist/conf/type/__systemd_unit/manifest @@ -29,6 +29,7 @@ fi name="${__object_id}" source=$(cat "${__object}/parameter/source") state=$(cat "${__object}/parameter/state") +enablement_state=$(cat "${__object}/parameter/enablement-state") # The unit must be disabled before removing its unit file. The unit file is # therefore removed by gencode-remote of this type, not here. @@ -41,8 +42,17 @@ if [ "${source}" = "-" ]; then source="${__object}/stdin" fi +unitfile_state="${state}" +if [ "${enablement_state}" = "masked" ]; then + # Masking creates a symlink from /etc/systemd/system/ to /dev/null. + # This process fails with "Failed to execute operation: Invalid argument" + # if file /etc/systemd/system/ already exists. We must therefore + # remove it. + unitfile_state="absent" +fi + __config_file "/etc/systemd/system/${name}" \ --mode 644 \ --onchange "systemctl daemon-reload" \ --source "${source}" \ - --state "${state}" + --state "${unitfile_state}" From 1f2b7c84c2c56b69267321174971794da2d1b986 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 18 Jul 2018 07:40:34 +0200 Subject: [PATCH 077/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 921e0c26..370c034b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) + * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) From f04f76524c60b8de865079e24a6143a83b8e5d7b Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sun, 22 Jul 2018 18:35:30 +0200 Subject: [PATCH 078/178] Bugfix: - __grafana_dashboard had the wrong release name for devuan ascii --- cdist/conf/type/__grafana_dashboard/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index e62bd15f..308af59a 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -9,7 +9,7 @@ case $os in 8*|jessie) apt_source_distribution=jessie ;; - 9*|ascii/ceres) + 9*|ascii/ceres|ascii) apt_source_distribution=stretch ;; *) From eeb059015029208ec55131caf2e5f17c3bd0b738 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 23 Jul 2018 08:09:55 +0200 Subject: [PATCH 079/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 370c034b..311d7a57 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) + * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) From 2f89d8a514a5996a76ca5a34397b3d087e2fe5c2 Mon Sep 17 00:00:00 2001 From: Stu Zhao Date: Tue, 28 Aug 2018 23:27:13 -0700 Subject: [PATCH 080/178] Fix __package_update_index processing error exit 1 in explorer will abort cdist. --- cdist/conf/type/__package_update_index/explorer/currage | 1 - docs/changelog | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage index cd042bd5..e86e3f13 100644 --- a/cdist/conf/type/__package_update_index/explorer/currage +++ b/cdist/conf/type/__package_update_index/explorer/currage @@ -29,6 +29,5 @@ case "$os" in ;; *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 - exit 1 ;; esac diff --git a/docs/changelog b/docs/changelog index 311d7a57..12bdb1e0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) + * Type __package_update_index: Fix error when using OS not using apt (Stu Zhao) 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) From b12c02138df4086e2e8e5e366e2156fa2ff95b3a Mon Sep 17 00:00:00 2001 From: Stu Zhao Date: Wed, 29 Aug 2018 00:07:19 -0700 Subject: [PATCH 081/178] Refactor __package_update_index explorers * add an type explorer to unify detecting of package type. * update currage use the type explorer, so if os and passed in type does not match, it behaves correctly. --- .../__package_update_index/explorer/currage | 8 ++--- .../type/__package_update_index/explorer/type | 34 +++++++++++++++++++ .../__package_update_index/gencode-remote | 18 +--------- 3 files changed, 39 insertions(+), 21 deletions(-) create mode 100644 cdist/conf/type/__package_update_index/explorer/type diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage index e86e3f13..45653f91 100644 --- a/cdist/conf/type/__package_update_index/explorer/currage +++ b/cdist/conf/type/__package_update_index/explorer/currage @@ -17,17 +17,17 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -os="$("$__explorer/os")" +type="$($__type_explorer/type)" -case "$os" in - debian|ubuntu|devuan) +case "$type" in + apt) if [ -f "/var/cache/apt/pkgcache.bin" ]; then echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin))) else echo 0 fi ;; - *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + *) echo "Your specified type ($type) is currently not supported." >&2 echo "Please contribute an implementation for it if you can." >&2 ;; esac diff --git a/cdist/conf/type/__package_update_index/explorer/type b/cdist/conf/type/__package_update_index/explorer/type new file mode 100644 index 00000000..805b9f04 --- /dev/null +++ b/cdist/conf/type/__package_update_index/explorer/type @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2018 Stu Zhao (z12y12l12 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +if [ -f "$__object/parameter/type" ]; then + cat "$__object/parameter/type" +else + # By default determine package manager based on operating system + os="$($__explorer/os)" + case "$os" in + amazon|scientific|centos|fedora|redhat) echo "yum" ;; + debian|ubuntu|devuan) echo "apt" ;; + archlinux) echo "pacman" ;; + *) + echo "Don't know how to manage packages on: $os" >&2 + exit 1 + ;; + esac +fi diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 37bfe7ab..132a3daa 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -21,28 +21,12 @@ # Update the package index with the appropriate package manager # -type="$__object/parameter/type" +type=$(cat "$__object/explorer/type") if [ -f "$__object/parameter/maxage" ]; then maxage="$(cat "$__object/parameter/maxage")" currage="$(cat "$__object/explorer/currage")" fi -if [ -f "$type" ]; then - type="$(cat "$type")" -else - # By default determine package manager based on operating system - os="$(cat "$__global/explorer/os")" - case "$os" in - amazon|scientific|centos|fedora|redhat) type="yum" ;; - debian|ubuntu|devuan) type="apt" ;; - archlinux) type="pacman" ;; - *) - echo "Don't know how to manage packages on: $os" >&2 - exit 1 - ;; - esac -fi - if [ -n "$maxage" ] && [ "$type" != "apt" ]; then echo "ERROR: \"--maxage\" only supported for \"apt\" pkg-manager." >&2 exit 1 From f143a70463e0740937370faa9492daf5cba9a2fd Mon Sep 17 00:00:00 2001 From: Stu Zhao Date: Wed, 29 Aug 2018 00:30:27 -0700 Subject: [PATCH 082/178] Support pacman with --maxage parameter --- .../__package_update_index/explorer/currage | 7 ++++ .../__package_update_index/gencode-remote | 35 ++++++++++--------- .../conf/type/__package_update_index/man.rst | 6 ++-- docs/changelog | 1 + 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage index 45653f91..50474fb3 100644 --- a/cdist/conf/type/__package_update_index/explorer/currage +++ b/cdist/conf/type/__package_update_index/explorer/currage @@ -27,6 +27,13 @@ case "$type" in echo 0 fi ;; + pacman) + if [ -d "/var/lib/pacman/sync" ]; then + echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync))) + else + echo 0 + fi + ;; *) echo "Your specified type ($type) is currently not supported." >&2 echo "Please contribute an implementation for it if you can." >&2 ;; diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 132a3daa..2cc76b81 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -22,32 +22,33 @@ # type=$(cat "$__object/explorer/type") +currage="$(cat "$__object/explorer/currage")" if [ -f "$__object/parameter/maxage" ]; then maxage="$(cat "$__object/parameter/maxage")" - currage="$(cat "$__object/explorer/currage")" fi -if [ -n "$maxage" ] && [ "$type" != "apt" ]; then - echo "ERROR: \"--maxage\" only supported for \"apt\" pkg-manager." >&2 - exit 1 +if [ -n "$maxage" ]; then + if [ "$type" != "apt" -a "$type" != "pacman" ]; then + echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2 + exit 1 + elif [ $currage -lt $maxage ]; then + exit 0 # no need to update + fi fi + case "$type" in yum) ;; - apt) if [ -n "$maxage" ]; then - ## check if we need to update: - if [ $currage -ge $maxage ]; then - echo "apt-get --quiet update" - echo "apt-cache updated (age was: $currage)" >> "$__messages_out" - fi - else - echo "apt-get --quiet update" - echo "apt-cache updated (age was: $currage)" >> "$__messages_out" - fi - ;; - pacman) echo "pacman --noprogressbar --sync --refresh" ;; + apt) + echo "apt-get --quiet update" + echo "apt-cache updated (age was: $currage)" >> "$__messages_out" + ;; + pacman) + echo "pacman --noprogressbar --sync --refresh" + echo "pacman package database synced (age was: $currage)" >> "$__messages_out" + ;; *) - echo "Don't know how to manage packages on: $os" >&2 + echo "Don't know how to manage packages for type: $type" >&2 exit 1 ;; esac diff --git a/cdist/conf/type/__package_update_index/man.rst b/cdist/conf/type/__package_update_index/man.rst index b63af654..3cd787b9 100644 --- a/cdist/conf/type/__package_update_index/man.rst +++ b/cdist/conf/type/__package_update_index/man.rst @@ -28,8 +28,8 @@ type * pacman for Arch Linux maxage - Available for package manager apt, max time in seconds since last update. - Repo update is skipped if maxage is not reached yet. + Available for package manager apt and pacman, max time in seconds since + last update. Repo update is skipped if maxage is not reached yet. MESSAGES -------- @@ -51,6 +51,7 @@ EXAMPLES # Only update every hour: __package_update_index --maxage 3600 --type apt + # same as above (on apt-type systems): __package_update_index --maxage 3600 @@ -58,6 +59,7 @@ AUTHORS ------- | Ricardo Catalinas Jiménez | Thomas Eckert +| Stu Zhao COPYING diff --git a/docs/changelog b/docs/changelog index 12bdb1e0..e989fb61 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) * Type __package_update_index: Fix error when using OS not using apt (Stu Zhao) + * Type __package_update_index: Support --maxage for type pacman (Stu Zhao) 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) From 93d877f5bff4d12510146051af032bc92e062fd3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 30 Aug 2018 20:22:51 +0200 Subject: [PATCH 083/178] Add nonparallel marker for __apt_source type. --- cdist/conf/type/__apt_source/nonparallel | 0 docs/changelog | 1 + 2 files changed, 1 insertion(+) create mode 100644 cdist/conf/type/__apt_source/nonparallel diff --git a/cdist/conf/type/__apt_source/nonparallel b/cdist/conf/type/__apt_source/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/docs/changelog b/docs/changelog index 311d7a57..e7c59952 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) + * Type __apt_source: Add nonparallel marker (Darko Poljak) 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) From 282647a88c3bbaf66db53f643796dd76976320a6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 31 Aug 2018 10:48:21 +0200 Subject: [PATCH 084/178] Fix changelog formatting. --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 23b0e352..7df1a38e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,7 +6,7 @@ next: * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) * Type __apt_source: Add nonparallel marker (Darko Poljak) - * Type __package_update_index: Fix error when using OS not using apt (Stu Zhao) + * Type __package_update_index: Fix error when using OS not using apt (Stu Zhao) * Type __package_update_index: Support --maxage for type pacman (Stu Zhao) 4.10.1: 2018-06-21 From a627247908bdce5b393a6296b71b5300f5fa6b30 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 1 Sep 2018 10:38:16 +0200 Subject: [PATCH 085/178] Check if certbot exists before using it. Fixes #685. --- .../__letsencrypt_cert/explorer/certificate-domains | 8 ++++++-- .../__letsencrypt_cert/explorer/certificate-exists | 10 ++++++++-- .../__letsencrypt_cert/explorer/certificate-is-test | 12 +++++++++--- 3 files changed, 23 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains index 367fda93..db605b63 100755 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains @@ -1,4 +1,8 @@ #!/bin/sh -e -certbot certificates --cert-name "${__object_id:?}" | grep ' Domains: ' | \ - cut -d ' ' -f 6- | tr ' ' '\n' +certbot_path=$("${__type_explorer}/certbot-path") +if [ -n "${certbot_path}" ] +then + certbot certificates --cert-name "${__object_id:?}" | grep ' Domains: ' | \ + cut -d ' ' -f 6- | tr ' ' '\n' +fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists index d2ea35cc..4e6f44db 100755 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists @@ -1,7 +1,13 @@ #!/bin/sh -e -if certbot certificates | grep -q " Certificate Name: ${__object_id:?}$"; then - echo yes +certbot_path=$("${__type_explorer}/certbot-path") +if [ -n "${certbot_path}" ] +then + if certbot certificates | grep -q " Certificate Name: ${__object_id:?}$"; then + echo yes + else + echo no + fi else echo no fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test index 6d7b0ae9..9b445059 100755 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test @@ -1,8 +1,14 @@ #!/bin/sh -e -if certbot certificates --cert-name "${__object_id:?}" | \ - grep -q 'INVALID: TEST_CERT'; then - echo yes +certbot_path=$("${__type_explorer}/certbot-path") +if [ -n "${certbot_path}" ] +then + if certbot certificates --cert-name "${__object_id:?}" | \ + grep -q 'INVALID: TEST_CERT'; then + echo yes + else + echo no + fi else echo no fi From b9ddbb1aa04dd2217db0603e6a201378a5e7b145 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 1 Sep 2018 23:30:53 +0200 Subject: [PATCH 086/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 7df1a38e..8a188501 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __apt_source: Add nonparallel marker (Darko Poljak) * Type __package_update_index: Fix error when using OS not using apt (Stu Zhao) * Type __package_update_index: Support --maxage for type pacman (Stu Zhao) + * Type __letsencrypt_cert: Fix explorers: check that certbot exists before using it (Darko Poljak) 4.10.1: 2018-06-21 * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) From 3782656569547b775f3b310ff6a226e49c0a7581 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 4 Sep 2018 08:27:20 +0200 Subject: [PATCH 087/178] Report encoding errors nicely. --- cdist/__init__.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index b7602ff8..000a571c 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -100,17 +100,23 @@ class CdistEntityError(Error): for name, path in stdpaths: if name not in result: result[name] = [] - if os.path.exists(path) and os.path.getsize(path) > 0: - output = [] - label_begin = name + ":" + header_name - output.append(label_begin) - output.append('\n') - output.append('-' * len(label_begin)) - output.append('\n') - with open(path, 'r') as fd: - output.append(fd.read()) - output.append('\n') - result[name].append(''.join(output)) + try: + if os.path.exists(path) and os.path.getsize(path) > 0: + output = [] + label_begin = name + ":" + header_name + output.append(label_begin) + output.append('\n') + output.append('-' * len(label_begin)) + output.append('\n') + with open(path, 'r') as fd: + output.append(fd.read()) + output.append('\n') + result[name].append(''.join(output)) + except UnicodeError as ue: + result[name].append(('Cannot output {}:{} due to: {}.\n' + 'You can try to read the error file "{}"' + ' yourself.').format( + name, header_name, ue, path)) return result def _stderr(self): From 55dcaecafe53cbc54b89346e3ab37ae85a3d7c6d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 6 Sep 2018 07:11:04 +0200 Subject: [PATCH 088/178] Release 4.10.2 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8a188501..c58e445b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.10.2: 2018-09-06 * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) From c2f8c7abb33ff971c30a00a29ef2c6f4016ebcca Mon Sep 17 00:00:00 2001 From: Adam Dej Date: Sat, 21 Jul 2018 14:52:50 +0200 Subject: [PATCH 089/178] Add explorer for info in /etc/os-release --- cdist/conf/explorer/os_release | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 cdist/conf/explorer/os_release diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release new file mode 100644 index 00000000..cfc01004 --- /dev/null +++ b/cdist/conf/explorer/os_release @@ -0,0 +1,26 @@ +#!/bin/sh +# +# 2018 Adam Dej (dejko.a at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# See os-release(5) and http://0pointer.de/blog/projects/os-release + +set +e + +cat /etc/os-release || cat /usr/lib/os-release || true From 2e653409c132f9869e962dc05c905f2204d9f0b7 Mon Sep 17 00:00:00 2001 From: Adam Dej Date: Sat, 21 Jul 2018 14:53:53 +0200 Subject: [PATCH 090/178] Update __docker type - Update installation method, following official installation instructions. - docker-engine was renamed to Docker CE around March 2017. Update manpage to reflect that change. - Remove flag `--experimental` since it is no longer necessary to install a different binary to get experimental features. Experimental features are included in the stable binary and can be enabled by a flag or in a config file. --- cdist/conf/type/__docker/man.rst | 16 ++---- cdist/conf/type/__docker/manifest | 66 ++++++++-------------- cdist/conf/type/__docker/parameter/boolean | 1 - 3 files changed, 28 insertions(+), 55 deletions(-) delete mode 100644 cdist/conf/type/__docker/parameter/boolean diff --git a/cdist/conf/type/__docker/man.rst b/cdist/conf/type/__docker/man.rst index 70b92cc7..5cb28ee1 100644 --- a/cdist/conf/type/__docker/man.rst +++ b/cdist/conf/type/__docker/man.rst @@ -3,12 +3,12 @@ cdist-type__docker(7) NAME ---- -cdist-type__docker - install docker-engine +cdist-type__docker - install Docker CE DESCRIPTION ----------- -Installs latest docker-engine package from dockerproject.org. +Installs latest Docker Community Edition package. REQUIRED PARAMETERS @@ -18,16 +18,13 @@ None. OPTIONAL PARAMETERS ------------------- -None. +state + 'present' or 'absent', defaults to 'present' BOOLEAN PARAMETERS ------------------ -experimental - Install the experimental docker-engine package instead of the latest stable release. - -state - 'present' or 'absent', defaults to 'present' +None. EXAMPLES @@ -38,9 +35,6 @@ EXAMPLES # Install docker __docker - # Install experimental - __docker --experimental - # Remove docker __docker --state absent diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index 1b1b1fb7..d501a9f1 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -24,57 +24,37 @@ state=$(cat "$__object/parameter/state") case "$os" in centos) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" + if (source "$__global/explorer/os_release" && [ "${VERSION_ID}" = "7" ]); then + __yum_repo docker-ce-stable \ + --name 'Docker CE Stable' \ + --baseurl "https://download.docker.com/linux/centos/7/\$basearch/stable" \ + --enabled \ + --gpgcheck 1 \ + --gpgkey 'https://download.docker.com/linux/centos/gpg' \ + --state ${state} + require="__yum_repo/docker-ce-stable" __package docker-ce --state ${state} + else + echo "CentOS version 7 is required!" >&2 + exit 1 fi - __yum_repo docker \ - --name 'Docker Repository' \ - --baseurl "https://yum.dockerproject.org/repo/$component/centos/\$releasever/" \ - --enabled \ - --gpgcheck 1 \ - --gpgkey 'https://yum.dockerproject.org/gpg' \ - --state ${state} - require="__yum_repo/docker" __package docker-engine --state ${state} ;; - ubuntu) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" + ubuntu|debian) + if [ "${state}" = "present" ]; then + __package apt-transport-https + __package ca-certificates + __package gnupg2 fi - __package apparmor --state ${state} - __package ca-certificates --state ${state} - __package apt-transport-https --state ${state} - __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D --state ${state} + __apt_key_uri docker --name "Docker Release (CE deb) " \ + --uri "https://download.docker.com/linux/${os}/gpg" --state ${state} export CDIST_ORDER_DEPENDENCY=on __apt_source docker \ - --uri https://apt.dockerproject.org/repo \ - --distribution "ubuntu-$(cat "$__global/explorer/lsb_codename")" \ + --uri "https://download.docker.com/linux/${os}" \ + --distribution "$(cat "$__global/explorer/lsb_codename")" \ --state ${state} \ - --component "$component" - __package docker-engine --state ${state} + --component "stable" + __package docker-ce --state ${state} unset CDIST_ORDER_DEPENDENCY ;; - debian) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" - fi - - __package apt-transport-https --state ${state} - __package ca-certificates --state ${state} - __package gnupg2 --state ${state} - __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D --state ${state} - export CDIST_ORDER_DEPENDENCY=on - __apt_source docker \ - --uri https://apt.dockerproject.org/repo \ - --distribution "debian-$(cat "$__global/explorer/lsb_codename")" \ - --state ${state} \ - --component "$component" - __package docker-engine --state ${state} - unset CDIST_ORDER_DEPENDENCY - - ;; *) echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 echo "Please contribute an implementation for it if you can." >&2 diff --git a/cdist/conf/type/__docker/parameter/boolean b/cdist/conf/type/__docker/parameter/boolean deleted file mode 100644 index 9839eb20..00000000 --- a/cdist/conf/type/__docker/parameter/boolean +++ /dev/null @@ -1 +0,0 @@ -experimental From 19a55ac75a9c555033b8c46e5987baee52dac685 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 9 Sep 2018 19:27:39 +0200 Subject: [PATCH 091/178] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index c58e445b..6c4a69a9 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * New global explorer: os_release (Ľubomír Kučera) + * Type __docker: Update type, install docker CE (Ľubomír Kučera) + 4.10.2: 2018-09-06 * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) From 346f1f8af34de7a6a8fd21dc521211d31b2f33c0 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Wed, 12 Sep 2018 11:36:06 +0200 Subject: [PATCH 092/178] Send messages when installing packages via __package_apt --- cdist/conf/type/__package_apt/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index d9cc52b7..39584e30 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -78,9 +78,11 @@ case "$state_should" in name="${name}=${version}" fi echo $aptget install $target_release \"$name\" + echo "installed" >> "$__messages_out" ;; absent) echo $aptget remove $purgeparam \"$name\" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 From b3f3b907a49759db18f95ed48b7890328a755e96 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Wed, 12 Sep 2018 11:35:23 +0200 Subject: [PATCH 093/178] Quote aptget command and params, simplify explorer --- cdist/conf/type/__package_apt/explorer/state | 2 +- cdist/conf/type/__package_apt/gencode-remote | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__package_apt/explorer/state b/cdist/conf/type/__package_apt/explorer/state index 658429ac..7ccd6fce 100755 --- a/cdist/conf/type/__package_apt/explorer/state +++ b/cdist/conf/type/__package_apt/explorer/state @@ -30,7 +30,7 @@ fi # Except dpkg failing, if package is not known / installed packages="$(apt-cache showpkg "$name" | sed -e "1,/Reverse Provides:/d" | cut -d ' ' -f 1) $name" for p in $packages; do - if [ -n "$(dpkg -s "$p" 2>/dev/null | grep "^Status: install ok installed$")" ]; then + if dpkg -s "$p" 2>/dev/null | grep --quiet "^Status: install ok installed$" ; then version=$(dpkg -s "$p" 2>/dev/null | grep "^Version:" | cut -d ' ' -f 2) echo "present $p $version" exit 0 diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index 39584e30..cbe14787 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -77,11 +77,11 @@ case "$state_should" in if [ -n "$version" ]; then name="${name}=${version}" fi - echo $aptget install $target_release \"$name\" + echo "$aptget install $target_release \"$name\"" echo "installed" >> "$__messages_out" ;; absent) - echo $aptget remove $purgeparam \"$name\" + echo "$aptget remove $purgeparam \"$name\"" echo "removed" >> "$__messages_out" ;; *) From aa80e8f87deaf4348296306ae26504fe19dbdd03 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 12 Sep 2018 11:53:46 +0200 Subject: [PATCH 094/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6c4a69a9..682544ff 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * New global explorer: os_release (Ľubomír Kučera) * Type __docker: Update type, install docker CE (Ľubomír Kučera) + * Type __package_apt: Write a message when a package is installed or removed; shellcheck (Jonas Weber) 4.10.2: 2018-09-06 * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) From d6952543a74e523a248342452cd4ecd4df81116b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 6 Sep 2018 07:06:25 +0200 Subject: [PATCH 095/178] Add 'real world example' walkthrough docs chapter. --- docs/changelog | 1 + docs/src/cdist-real-world.rst | 573 ++++++++++++++++++++++++++++++++++ docs/src/index.rst | 1 + 3 files changed, 575 insertions(+) create mode 100644 docs/src/cdist-real-world.rst diff --git a/docs/changelog b/docs/changelog index 682544ff..62d44fbe 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * New global explorer: os_release (Ľubomír Kučera) * Type __docker: Update type, install docker CE (Ľubomír Kučera) * Type __package_apt: Write a message when a package is installed or removed; shellcheck (Jonas Weber) + * Documentation: Add 'Dive into real world cdist' walkthrough chapter (Darko Poljak) 4.10.2: 2018-09-06 * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst new file mode 100644 index 00000000..8ccb0fc9 --- /dev/null +++ b/docs/src/cdist-real-world.rst @@ -0,0 +1,573 @@ +Dive into real world cdist +========================== + +Introduction +------------ + +This walkthrough shows real world cdist configuration example. + +Sample target host is named **test.ungleich.ch**. +Just replace **test.ungleich.ch** with your target hostname. + +Our goal is to configure python application hosting. For writing sample +application we will use `Bottle `_ WSGI micro web-framework. +It will use PostgreSQL database and it will list items from **items** table. +It will be served by uWSGI server. We will also use the Nginx web server +as a reverse proxy and we want HTTPS. +For HTTPS we will use Let's Encrypt certificate. + +For setting up hosting we want to use cdist so we will write a new type +for that. This type will: + +- install required packages +- create OS user, user home directory and application home directory +- create PostgreSQL database +- configure uWSGI +- configure Let's Encrypt certificate +- configure nginx. + +Our type will not create the actual python application. Its intention is only +to configure hosing for specified user and project. It is up to the user to +create his/her applications. + +So let's start. + +Creating type layout +-------------------- + +We will create a new custom type. Let's call it **__sample_bottle_hosting**. + +Go to **~/.cdist/type** directory (create it if it does not exist) and create +new type layout:: + + cd ~/.cdist/type + mkdir __sample_bottle_hosting + cd __sample_bottle_hosting + touch manifest gencode-remote + mkdir parameter + touch parameter/required + +Creating __sample_bottle_hosting type parameters +------------------------------------------------ + +Our type will be configurable through the means of parameters. Let's define +the following parameters: + +projectname + name for the project, needed for uWSGI ini file + +user + user name + +domain + target host domain, needed for Let's Encrypt certificate. + +We define parameters to make our type reusable for different projects, user and domain. + +Define required parameters:: + + printf "projectname\n" >> parameter/required + printf "user\n" >> parameter/required + printf "domain\n" >> parameter/required + +For details on type parameters see `Defining parameters `_. + +Creating __sample_bottle_hosting type manifest +---------------------------------------------- + +Next step is to define manifest (~/.cdist/type/__sample_bottle_hosting/manifest). +We also want our type to currently support only Devuan. So we will start by +checking target host OS. We will use `os `_ +global explorer:: + + os=$(cat "$__global/explorer/os") + + case "$os" in + devuan) + : + ;; + *) + echo "OS $os currently not supported" >&2 + exit 1 + ;; + esac + +If target host OS is not Devuan then we print error message to stderr +and exit. For other OS-es support we should check and change package names +we should install, because packages differ in different OS-es and in different +OS distributions like GNU/Linux distributions. There can also be a different +configuration locations (e.g. nginx config directory could be in /usr/local tree). +If we detected unsupported OS we should error out. cdist will stop configuration +process and output error message. + +Creating user and user directories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Then we create user and his/her home directory and application home directory. +We will use existing cdist types `__user `_ and `__directory `_:: + + user="$(cat "$__object/parameter/user")" + home="/home/$user" + apphome="$home/app" + + # create user + __user "$user" --home "$home" --shell /bin/bash + # create user home dir + require="__user/$user" __directory "$home" \ + --owner "$user" --group "$user" --mode 0755 + # create app home dir + require="__user/$user __directory/$home" __directory "$apphome" \ + --state present --owner "$user" --group "$user" --mode 0755 + +First we define *user*, *home* and *apphome* variables. User is defined by type's +**user** parameter. Here we use **require** which is cdist's way to define dependencies. +User home directory should be created **after** user is created. And application +home directory is created **after** both user and user home directory are created. +For details on **require** see `Dependencies `_. + +Installing packages +~~~~~~~~~~~~~~~~~~~ + +Install required packages using existing `__package `_ type. +Before installing package we want to update apt package index using +`__apt_update_index `_:: + + # define packages that need to be installed + packages_to_install="nginx uwsgi-plugin-python3 python3-dev python3-pip postgresql postgresql-contrib libpq-dev python3-venv uwsgi python3-psycopg2" + + # update package index + __apt_update_index + # install packages + for package in $packages_to_install + do require="__apt_update_index" __package $package --state=present + done + +Here we use shell for loop. It executes **require="__apt_update_index" __package** +for each member in a list we define in **packages_to_install** variable. +This is much nicer then having as many **require="__apt_update_index" __package** +lines as there are packages we want to install. + +For python packages we use `__package_pip `_:: + + # install pip3 packages + for package in bottle bottle-pgsql; do + __package_pip --pip pip3 $package + done + +Creating PostgreSQL database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create PostgreSQL database using `__postgres_database `_ +and `__postgres_role `_ for creating database user:: + + #PostgreSQL db & user + postgres_server=postgresql + + # create PostgreSQL db user + require="__package/postgresql" __postgres_role $user --login --createdb + # create PostgreSQL db + require="__postgres_role/$user __package/postgresql" __postgres_database $user \ + --owner $user + +Configuring uWSGI +~~~~~~~~~~~~~~~~~ + +Configure uWSGI using `__file `_ type:: + + # configure uWSGI + projectname="$(cat "$__object/parameter/projectname")" + require="__package/uwsgi" __file /etc/uwsgi/apps-enabled/$user.ini \ + --owner root --group root --mode 0644 \ + --state present \ + --source - << EOF + [uwsgi] + socket = $apphome/uwsgi.sock + chdir = $apphome + wsgi-file = $projectname/wsgi.py + touch-reload = $projectname/wsgi.py + processes = 4 + threads = 2 + chmod-socket = 666 + daemonize=true + vacuum = true + uid = $user + gid = $user + EOF + +We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$user.ini** file. +Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**. +If this directory does not exist then **__file** type would error. +We also use stdin as file content source. For details see `Input from stdin `_. +For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent +lines read by the shell to the input of a command until a line containing only the delimiter +and a newline, with no blank characters in between (EOF in our case). + +Configuring nginx for Let's Encrypt and HTTPS redirection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Next configure nginx for Let's Encrypt and for HTTP -> HTTPS redirection. For this +purpose we will create new type **__sample_nginx_http_letsencrypt_and_ssl_redirect** +and use it here:: + + domain="$(cat "$__object/parameter/domain")" + webroot="/var/www/html" + __sample_nginx_http_letsencrypt_and_ssl_redirect "$domain" --webroot "$webroot" + +Configuring certificate creation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +After HTTP nginx configuration we will create Let's Encrypt certificate using +`__letsencrypt_cert `_ type. +For Let's Encrypt cert configuration ensure that there is a DNS entry for your +domain. We assure that cert creation is applied after nginx HTTP is configured +for Let's Encrypt to work:: + + # create SSL cert + require="__package/nginx __sample_nginx_http_letsencrypt_and_ssl_redirect/$domain" \ + __letsencrypt_cert --admin-email admin@test.ungleich.ch \ + --webroot "$webroot" \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --domain "$domain" \ + "$domain" + +Configuring nginx HTTPS server with uWSGI upstream +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Then we can configure nginx HTTPS server that will use created Let's Encrypt certificate:: + + # configure nginx + require="__package/nginx __letsencrypt_cert/$domain" \ + __file "/etc/nginx/sites-enabled/https-$domain" \ + --source - --mode 0644 << EOF + upstream _bottle { + server unix:$apphome/uwsgi.sock; + } + + server { + listen 443; + listen [::]:443; + + server_name $domain; + + access_log /var/log/nginx/access.log; + + ssl on; + ssl_certificate /etc/letsencrypt/live/$domain/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/$domain/privkey.pem; + + client_max_body_size 256m; + + location / { + try_files \$uri @uwsgi; + } + + location @uwsgi { + include uwsgi_params; + uwsgi_pass _bottle; + } + } + EOF + +Now our manifest is finished. + +Complete __sample_bottle_hosting type manifest listing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here is complete __sample_bottle_hosting type manifest listing, +located in ~/.cdist/type/__sample_bottle_hosting/manifest:: + + #!/bin/sh + + os=$(cat "$__global/explorer/os") + + case "$os" in + devuan) + : + ;; + *) + echo "OS $os currently not supported" >&2 + exit 1 + ;; + esac + + projectname="$(cat "$__object/parameter/projectname")" + user="$(cat "$__object/parameter/user")" + home="/home/$user" + apphome="$home/app" + domain="$(cat "$__object/parameter/domain")" + + # create user + __user "$user" --home "$home" --shell /bin/bash + # create user home dir + require="__user/$user" __directory "$home" \ + --owner "$user" --group "$user" --mode 0755 + # create app home dir + require="__user/$user __directory/$home" __directory "$apphome" \ + --state present --owner "$user" --group "$user" --mode 0755 + + # define packages that need to be installed + packages_to_install="nginx uwsgi-plugin-python3 python3-dev python3-pip postgresql postgresql-contrib libpq-dev python3-venv uwsgi python3-psycopg2" + + # update package index + __apt_update_index + # install packages + for package in $packages_to_install + do require="__apt_update_index" __package $package --state=present + done + # install pip3 packages + for package in bottle bottle-pgsql; do + __package_pip --pip pip3 $package + done + + #PostgreSQL db & user + postgres_server=postgresql + + # create PostgreSQL db user + require="__package/postgresql" __postgres_role $user --login --createdb + # create PostgreSQL db + require="__postgres_role/$user __package/postgresql" __postgres_database $user \ + --owner $user + # configure uWSGI + require="__package/uwsgi" __file /etc/uwsgi/apps-enabled/$user.ini \ + --owner root --group root --mode 0644 \ + --state present \ + --source - << EOF + [uwsgi] + socket = $apphome/uwsgi.sock + chdir = $apphome + wsgi-file = $projectname/wsgi.py + touch-reload = $projectname/wsgi.py + processes = 4 + threads = 2 + chmod-socket = 666 + daemonize=true + vacuum = true + uid = $user + gid = $user + EOF + + # setup nginx HTTP for Let's Encrypt and SSL redirect + domain="$(cat "$__object/parameter/domain")" + webroot="/var/www/html" + __sample_nginx_http_letsencrypt_and_ssl_redirect "$domain" --webroot "$webroot" + + # create SSL cert + require="__package/nginx __sample_nginx_http_letsencrypt_and_ssl_redirect/$domain" \ + __letsencrypt_cert --admin-email admin@test.ungleich.ch \ + --webroot "$webroot" \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --domain "$domain" \ + "$domain" + + # configure nginx + require="__package/nginx __letsencrypt_cert/$domain" \ + __file "/etc/nginx/sites-enabled/https-$domain" \ + --source - --mode 0644 << EOF + upstream _bottle { + server unix:$apphome/uwsgi.sock; + } + + server { + listen 443; + listen [::]:443; + + server_name $domain; + + access_log /var/log/nginx/access.log; + + ssl on; + ssl_certificate /etc/letsencrypt/live/$domain/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/$domain/privkey.pem; + + client_max_body_size 256m; + + location / { + try_files \$uri @uwsgi; + } + + location @uwsgi { + include uwsgi_params; + uwsgi_pass _bottle; + } + } + EOF + +Creating __sample_bottle_hosting type gencode-remote +---------------------------------------------------- + +Now define **gencode-remote** script: ~/.cdist/type/__sample_bottle_hosting/gencode-remote. +After manifest is applied it should restart uWSGI and nginx services so that our +configuration is active. Our gencode-remote looks like the following:: + + echo "service uwsgi restart" + echo "service nginx restart" + +Our **__sample_bottle_hosting** type is now finished. + +Creating __sample_nginx_http_letsencrypt_and_ssl_redirect type +-------------------------------------------------------------- + +Let's now create **__sample_nginx_http_letsencrypt_and_ssl_redirect** type:: + + cd ~/.cdist/type + mkdir __sample_nginx_http_letsencrypt_and_ssl_redirect + cd __sample_nginx_http_letsencrypt_and_ssl_redirect + mkdir parameter + echo webroot > parameter/required + touch manifest + touch gencode-remote + +Edit manifest:: + + domain="$__object_id" + webroot="$(cat "$__object/parameter/webroot")" + # make sure we have nginx package + __package nginx + # setup Let's Encrypt HTTP acme challenge, redirect HTTP to HTTPS + require="__package/nginx" __file "/etc/nginx/sites-enabled/http-$domain" \ + --source - --mode 0644 << EOF + server { + listen *:80; + listen [::]:80; + + server_name $domain; + + # Let's Encrypt + location /.well-known/acme-challenge/ { + root $webroot; + } + + # Everything else -> SSL + location / { + return 301 https://\$host\$request_uri; + } + } + + EOF + +Edit gencode-remote:: + + echo "service nginx reload" + +Creating init manifest +---------------------- + +Next create init manifest:: + + cd ~/.cdist/manifest + printf "__sample_bottle_hosting --projectname sample --user app --domain \$__target_host sample\n" > sample + +Using this init manifest our target host will be configured using our **__sample_bottle_hosting** +type with projectname *sample*, user *app* and domain equal to **__target_host**. +Here the last positional argument *sample* is type's object id. For details on +**__target_host** and **__object_id** see +`Environment variables (for reading) `_ +reference. + +Configuring host +---------------- + +Finally configure test.ungleich.ch:: + + cdist config -v -i ~/.cdist/manifest/sample test.ungleich.ch + +After cdist configuration is successfully finished our host is ready. + +Creating python bottle application +---------------------------------- + +We now need to create Bottle application. As you remember from the beginning +of this walkthrough our type does not create the actual python application, +its intention is only to configure hosing for specified user and project. +It is up to the user to create his/her applications. + +Become app user:: + + su -l app + +Preparing database +~~~~~~~~~~~~~~~~~~ + +We need to prepare database for our application. Create table and +insert some items:: + + psql -c "create table items (item varchar(255));" + + psql -c "insert into items(item) values('spam');" + psql -c "insert into items(item) values('eggs');" + psql -c "insert into items(item) values('sausage');" + +Creating application +~~~~~~~~~~~~~~~~~~~~ + +Next create sample app:: + + cd /home/app/app + mkdir sample + cd sample + +Create app.py with the following content:: + + #!/usr/bin/env python3 + + import bottle + import bottle_pgsql + + app = application = bottle.Bottle() + plugin = bottle_pgsql.Plugin('dbname=app user=app password=') + app.install(plugin) + + @app.route('/') + def show_index(db): + db.execute('select * from items') + items = db.fetchall() or [] + rv = '

Items:

    ' + for item in items: + rv += '
  • ' + str(item['item']) + '
    • ' + rv += '
' + return rv + + if __name__ == '__main__': + bottle.run(app=app, host='0.0.0.0', port=8080) + +Create wsgi.py with the following content:: + + import os + + os.chdir(os.path.dirname(__file__)) + + import app + application = app.app + +We have configured uWSGI with **touch-reload = $projectname/wsgi.py** so after +we have changed our **wsgi.py** file uWSGI reloads the application. + +Our application selects and lists items from **items** table. + +Openning application +~~~~~~~~~~~~~~~~~~~~ + +Finally try the application:: + + http://test.ungleich.ch/ + +It should redirect to HTTPS and return: + +.. container:: highlight + + .. raw:: html + +

Items:

+ +
    +
  • spam
    • +
  • eggs
    • +
  • sausage
    • +
+ +What's next? +------------ + +Continue reading next sections ;) diff --git a/docs/src/index.rst b/docs/src/index.rst index 5d0bb537..bef91e1c 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -16,6 +16,7 @@ Contents: cdist-support cdist-features cdist-quickstart + cdist-real-world man1/cdist cdist-bootstrap cdist-configuration From f918acd725f911aa71afc14af5bf38d9c44fa6cc Mon Sep 17 00:00:00 2001 From: dhivael Date: Fri, 21 Sep 2018 22:54:37 +0200 Subject: [PATCH 096/178] don't duplicate Remote::mkdir in explorer transfer Remote::transfer contains a call to mkdir(destination) if the source is a directory. since the destination must also be a directory we can omit extra mkdir calls in Explorer. this saves about 10% on my manifests in sequential mode, and about 6% in parallel mode. --- cdist/core/explorer.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 8c18a19f..a42b0117 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -147,7 +147,6 @@ class Explorer(object): def transfer_global_explorers(self): """Transfer the global explorers to the remote side.""" - self.remote.mkdir(self.remote.global_explorer_path) self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path, self.jobs) @@ -223,7 +222,6 @@ class Explorer(object): cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) - self.remote.mkdir(destination) self.remote.transfer(source, destination) self.remote.run(["chmod", "0700", "%s/*" % (destination)]) self._type_explorers_transferred.append(cdist_type.name) @@ -235,5 +233,4 @@ class Explorer(object): cdist_object.parameter_path) destination = os.path.join(self.remote.object_path, cdist_object.parameter_path) - self.remote.mkdir(destination) self.remote.transfer(source, destination) From c021cd15cdac8d7a474107531df71dd211c1c6d4 Mon Sep 17 00:00:00 2001 From: dhivael Date: Fri, 21 Sep 2018 23:17:41 +0200 Subject: [PATCH 097/178] transfer all files of a directory at once instead of calling the copy command once per file in a directory (eg a type explorer dir), call the copy command only once with all files of the directory. batch copy saves 20% of dry-run time on my test manifest in sequential mode, and 15% in parallel mode. --- cdist/exec/remote.py | 40 +++++++--------------------------------- 1 file changed, 7 insertions(+), 33 deletions(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 4466545e..ffb3ee00 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -203,46 +203,20 @@ class Remote(object): os.remove(tarpath) used_archiving = True if not used_archiving: - if jobs: - self._transfer_dir_parallel(source, destination, jobs) - else: - self._transfer_dir_sequential(source, destination) + self._transfer_dir(source, destination) elif jobs: raise cdist.Error("Source {} is not a directory".format(source)) else: self._transfer_file(source, destination) - def _transfer_dir_commands(self, source, destination): + def _transfer_dir(self, source, destination): + command = self._copy.split() for f in glob.glob1(source, '*'): - command = self._copy.split() path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - yield command - - def _transfer_dir_sequential(self, source, destination): - for command in self._transfer_dir_commands(source, destination): - self._run_command(command) - - def _transfer_dir_parallel(self, source, destination, jobs): - """Transfer a directory to the remote side in parallel mode.""" - self.log.debug("Remote transfer in {} parallel jobs".format( - jobs)) - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for parallel " - "remote transfer")) - args = [ - (command, ) - for command in self._transfer_dir_commands(source, destination) - ] - if len(args) == 1: - self.log.debug("Only one dir entry, transfering sequentially") - self._run_command(args[0]) - else: - mp_pool_run(self._run_command, args, jobs=jobs) - self.log.trace(("Multiprocessing for parallel transfer " - "finished")) + command.extend([path]) + command.extend(['{0}:{1}'.format( + _wrap_addr(self.target_host[0]), destination)]) + self._run_command(command) def run_script(self, script, env=None, return_output=False, stdout=None, stderr=None): From 1f703ab42ef4a1bc1730577c213cef5f980e921d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 22 Sep 2018 20:52:43 +0200 Subject: [PATCH 098/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 62d44fbe..d25c4462 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Type __docker: Update type, install docker CE (Ľubomír Kučera) * Type __package_apt: Write a message when a package is installed or removed; shellcheck (Jonas Weber) * Documentation: Add 'Dive into real world cdist' walkthrough chapter (Darko Poljak) + * Core: Remove duplicate remote mkdir calls in explorer transfer (myeisha) 4.10.2: 2018-09-06 * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) From 02982ca8bee500a0d0cdf1975074953278f4109e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Sep 2018 12:04:57 +0200 Subject: [PATCH 099/178] Release 4.10.3 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index d25c4462..8c310a22 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) * Type __docker: Update type, install docker CE (Ľubomír Kučera) * Type __package_apt: Write a message when a package is installed or removed; shellcheck (Jonas Weber) From 6acf6f64dcf6a5768be6f2874a719b16b5ba6a4f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Sep 2018 20:34:42 +0200 Subject: [PATCH 100/178] Add timestamp to log messages. --- cdist/argparse.py | 5 +++++ cdist/config.py | 8 +++++++- cdist/log.py | 28 ++++++++++++++++++++++++++++ docs/src/man1/cdist.rst | 10 ++++++++-- 4 files changed, 48 insertions(+), 3 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 29620751..0ee9d7b1 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -251,6 +251,11 @@ def get_parsers(): 'line). If no host or host file is specified then, by ' 'default, read hosts from stdin.'), dest='hostfile', required=False) + parser['config_args'].add_argument( + '-P', '--timestamp', + help=('Timestamp log messages with the current local date and time ' + 'in the format: YYYYMMDDHHMMSS.us.'), + action='store_true', dest='timestamp') parser['config_args'].add_argument( '-p', '--parallel', nargs='?', metavar='HOST_MAX', type=functools.partial(check_lower_bounded_int, lower_bound=1, diff --git a/cdist/config.py b/cdist/config.py index 2dcb1005..bfb89419 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -175,7 +175,13 @@ class Config(object): if args.parallel or args.jobs: # If parallel execution then also log process id - cdist.log.setupParallelLogging() + if args.timestamp: + cdist.log.setupTimestampingParallelLogging() + else: + cdist.log.setupParallelLogging() + log = logging.getLogger("cdist") + elif args.timestamp: + cdist.log.setupTimestampingLogging() log = logging.getLogger("cdist") if args.parallel: diff --git a/cdist/log.py b/cdist/log.py index dba1ad2f..5d431130 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -22,6 +22,7 @@ import logging import sys +import datetime # Define additional cdist logging levels. @@ -95,15 +96,42 @@ class DefaultLog(logging.Logger): self.log(logging.TRACE, msg, *args, **kwargs) +class TimestampingLog(DefaultLog): + + def filter(self, record): + """Add timestamp to messages""" + + super().filter(record) + now = datetime.datetime.now() + timestamp = now.strftime("%Y%m%d%H%M%S.%f") + record.msg = "[" + timestamp + "] " + str(record.msg) + + return True + + class ParallelLog(DefaultLog): FORMAT = '%(levelname)s: [%(process)d]: %(message)s' +class TimestampingParallelLog(TimestampingLog, ParallelLog): + pass + + def setupDefaultLogging(): del logging.getLogger().handlers[:] logging.setLoggerClass(DefaultLog) +def setupTimestampingLogging(): + del logging.getLogger().handlers[:] + logging.setLoggerClass(TimestampingLog) + + +def setupTimestampingParallelLogging(): + del logging.getLogger().handlers[:] + logging.setLoggerClass(TimestampingParallelLog) + + def setupParallelLogging(): del logging.getLogger().handlers[:] logging.setLoggerClass(ParallelLog) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 79683883..0a628660 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -20,7 +20,8 @@ SYNOPSIS [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-4] [-6] - [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-S] [-s] [-t] + [-A] [-a] [-f HOSTFILE] [-P] [-p [HOST_MAX]] [-S] [-s] + [-t] [host [host ...]] cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] @@ -28,7 +29,8 @@ SYNOPSIS [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-4] [-6] - [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-S] [-s] [-t] + [-A] [-a] [-f HOSTFILE] [-P] [-p [HOST_MAX]] [-S] [-s] + [-t] [host [host ...]] cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... @@ -171,6 +173,10 @@ Install command is currently in beta. **-o OUT_PATH, --out-dir OUT_PATH** Directory to save cdist output in. +**-P, --timestamp** + Timestamp log messages with the current local date and time + in the format: YYYYMMDDHHMMSS.us. + **-p [HOST_MAX], --parallel [HOST_MAX]** Operate on multiple hosts in parallel for specified maximum hosts at a time. Without argument CPU count is From 0c57b3083ec4f43ff6fc56345a3007cfbd4dc3f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Sep 2018 21:38:03 +0200 Subject: [PATCH 101/178] Fix getting loggers. --- cdist/config.py | 21 ++++++++------------- cdist/core/cdist_type.py | 2 +- scripts/cdist | 3 +-- 3 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index bfb89419..e8fd5384 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -165,25 +165,20 @@ class Config(object): def commandline(cls, args): """Configure remote system""" - # FIXME: Refactor relict - remove later - log = logging.getLogger("cdist") + if (args.parallel and args.parallel != 1) or args.jobs: + if args.timestamp: + cdist.log.setupTimestampingParallelLogging() + else: + cdist.log.setupParallelLogging() + elif args.timestamp: + cdist.log.setupTimestampingLogging() + log = logging.getLogger("config") # No new child process if only one host at a time. if args.parallel == 1: log.debug("Only 1 parallel process, doing it sequentially") args.parallel = 0 - if args.parallel or args.jobs: - # If parallel execution then also log process id - if args.timestamp: - cdist.log.setupTimestampingParallelLogging() - else: - cdist.log.setupParallelLogging() - log = logging.getLogger("cdist") - elif args.timestamp: - cdist.log.setupTimestampingLogging() - log = logging.getLogger("cdist") - if args.parallel: import signal diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 40194f94..99e40e70 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -47,7 +47,7 @@ class CdistType(object): """ - log = logging.getLogger("cdist") + log = logging.getLogger("cdist-type") def __init__(self, base_path, name): self.base_path = base_path diff --git a/scripts/cdist b/scripts/cdist index 088e4dc2..3110e657 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -69,8 +69,6 @@ if __name__ == "__main__": import re import os - log = logging.getLogger("cdist") - if re.match("__", os.path.basename(sys.argv[0])): import cdist.emulator emulator = cdist.emulator.Emulator(sys.argv) @@ -82,6 +80,7 @@ if __name__ == "__main__": exit_code = 2 except cdist.Error as e: + log = logging.getLogger("cdist") log.error(e) exit_code = 1 From c04325fdc5e2fb96aa7d3fbb26a06f13ec7e75ce Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Sep 2018 22:26:16 +0200 Subject: [PATCH 102/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 8c310a22..6d2ef147 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) + 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) * Type __docker: Update type, install docker CE (Ľubomír Kučera) From e404ff4836f91956d7367363bf6217d3af623b3d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 26 Sep 2018 20:35:32 +0200 Subject: [PATCH 103/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6d2ef147..8fd2ea49 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) + * Core: Add timestamp (optional) to log messages (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 82077aff251c9d13e1aac4608ba7bf0a58ea5712 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 27 Sep 2018 18:55:33 +0200 Subject: [PATCH 104/178] Add timestamping log option to config file. --- cdist/configuration.py | 2 + cdist/test/configuration/__init__.py | 112 +++++++++++++++++++++++++++ docs/src/cdist-configuration.rst | 4 + docs/src/man1/cdist.rst | 4 + 4 files changed, 122 insertions(+) diff --git a/cdist/configuration.py b/cdist/configuration.py index 8df43f9f..f05a5963 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -261,6 +261,7 @@ _ARG_OPTION_MAPPING = { 'verbose': 'verbosity', 'use_archiving': 'archiving', 'save_output_streams': 'save_output_streams', + 'timestamp': 'timestamp', } @@ -304,6 +305,7 @@ class Configuration(metaclass=Singleton): 'archiving': ArchivingOption(), 'save_output_streams': BooleanOption('save_output_streams', default_overrides=False), + 'timestamp': BooleanOption('timestamp'), }, } diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 3319d320..182868a6 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -1296,6 +1296,118 @@ class ConfigurationTestCase(test.CdistTestCase): self.fail("Exception should not have been raised: {}".format( e)) + def test_configuration_timestamping_log_1(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'True', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': True, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = True + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_timestamping_log_2(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'False', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': True, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = True + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_timestamping_log_3(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'False', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = False + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_timestamping_log_4(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'True', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = False + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + if __name__ == "__main__": import unittest diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst index cf1e373b..4c9b4d33 100644 --- a/docs/src/cdist-configuration.rst +++ b/docs/src/cdist-configuration.rst @@ -94,6 +94,10 @@ The possible keywords and their meanings are as follows: It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'. +:strong:`timestamp` + Timestamp log messages with the current local date and time + in the format: YYYYMMDDHHMMSS.us. + :strong:`verbosity` Set verbosity level. Valid values are: 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0a628660..3a74f1ef 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -521,6 +521,10 @@ The possible keywords and their meanings are as follows: It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'. +:strong:`timestamp` + Timestamp log messages with the current local date and time + in the format: YYYYMMDDHHMMSS.us. + :strong:`verbosity` Set verbosity level. Valid values are: 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. From 9cb9771e3347a7ab5830021822c04121130bfb29 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 29 Sep 2018 11:15:20 +0200 Subject: [PATCH 105/178] Exclude man7 empty directory from archive. --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 76eac516..aeaab8b9 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,3 +3,4 @@ .gitkeep export-ignore docs/speeches export-ignore docs/video export-ignore +docs/src/man7 export-ignore From e551348cd74be0c6506af183fa242021567b2849 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 29 Sep 2018 11:15:48 +0200 Subject: [PATCH 106/178] Create man7 directory if needed. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index d727bccc..3a1819be 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,7 @@ MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX)) # Link manpage: do not create man.html but correct named file $(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst + mkdir -p $(MAN7DSTDIR) ln -sf "../../../$^" $@ # Manpages #2: reference From 6326eae1bb724a2ecb8eb45099472f93c9852cbf Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Mon, 1 Oct 2018 22:13:24 +0200 Subject: [PATCH 107/178] Fix SC2166 on all manfest/gencode/explorers --- cdist/conf/type/__ccollect_source/gencode-remote | 2 +- cdist/conf/type/__consul_agent/manifest | 2 +- cdist/conf/type/__consul_check/manifest | 2 +- cdist/conf/type/__consul_service/manifest | 6 +++--- cdist/conf/type/__consul_template_template/manifest | 4 ++-- cdist/conf/type/__consul_watch_checks/manifest | 2 +- cdist/conf/type/__file/gencode-local | 6 +++--- cdist/conf/type/__file/gencode-remote | 2 +- cdist/conf/type/__git/gencode-remote | 12 ++++++------ cdist/conf/type/__hostname/gencode-remote | 4 ++-- .../__install_partition_msdos_apply/gencode-remote | 2 +- cdist/conf/type/__jail_freebsd10/gencode-remote | 4 ++-- cdist/conf/type/__jail_freebsd9/gencode-remote | 4 ++-- cdist/conf/type/__key_value/manifest | 2 +- cdist/conf/type/__line/gencode-remote | 2 +- cdist/conf/type/__link/gencode-remote | 2 +- cdist/conf/type/__package/manifest | 2 +- cdist/conf/type/__package_emerge/gencode-remote | 4 ++-- cdist/conf/type/__package_pkg_freebsd/gencode-remote | 2 +- cdist/conf/type/__package_pkg_openbsd/gencode-remote | 2 +- .../conf/type/__package_update_index/gencode-remote | 2 +- cdist/conf/type/__pyvenv/gencode-remote | 12 ++++++------ cdist/conf/type/__ssh_authorized_keys/manifest | 2 +- 23 files changed, 42 insertions(+), 42 deletions(-) diff --git a/cdist/conf/type/__ccollect_source/gencode-remote b/cdist/conf/type/__ccollect_source/gencode-remote index 56003fef..763f219e 100755 --- a/cdist/conf/type/__ccollect_source/gencode-remote +++ b/cdist/conf/type/__ccollect_source/gencode-remote @@ -71,7 +71,7 @@ case "$state_should" in fi value_is="$(get_current_value "$attribute" "$value_should")" - if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then + if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then "set_$attribute" "$value_should" fi fi diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index 820018c9..a696894b 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -66,7 +66,7 @@ require="__directory/etc/consul" \ __directory "$conf_dir" \ --owner root --group "$group" --mode 750 --state "$state" -if [ -f "$__object/parameter/ca-file-source" -o -f "$__object/parameter/cert-file-source" -o -f "$__object/parameter/key-file-source" ]; then +if [ -f "$__object/parameter/ca-file-source" ] || [ -f "$__object/parameter/cert-file-source" ] || [ -f "$__object/parameter/key-file-source" ]; then # create directory for ssl certs require="__directory/etc/consul" \ __directory /etc/consul/ssl \ diff --git a/cdist/conf/type/__consul_check/manifest b/cdist/conf/type/__consul_check/manifest index 8149b130..554c0680 100755 --- a/cdist/conf/type/__consul_check/manifest +++ b/cdist/conf/type/__consul_check/manifest @@ -40,7 +40,7 @@ if [ ! -f "$__object/parameter/interval" ]; then fi done fi -if [ -f "$__object/parameter/docker-container-id" -a ! -f "$__object/parameter/script" ]; then +if [ -f "$__object/parameter/docker-container-id" ] && [ ! -f "$__object/parameter/script" ]; then echo "When using --docker-container-id you must also define --script." >&2 exit 1 fi diff --git a/cdist/conf/type/__consul_service/manifest b/cdist/conf/type/__consul_service/manifest index d7a1b6e3..f6d29b4a 100755 --- a/cdist/conf/type/__consul_service/manifest +++ b/cdist/conf/type/__consul_service/manifest @@ -24,15 +24,15 @@ conf_file="service_${name}.json" state="$(cat "$__object/parameter/state")" # Sanity checks -if [ -f "$__object/parameter/check-script" -a -f "$__object/parameter/check-ttl" ]; then +if [ -f "$__object/parameter/check-script" ] && [ -f "$__object/parameter/check-ttl" ]; then echo "Use either --check-script together with --check-interval OR --check-ttl, but not both" >&2 exit 1 fi -if [ -f "$__object/parameter/check-script" -a ! -f "$__object/parameter/check-interval" ]; then +if [ -f "$__object/parameter/check-script" ] && [ ! -f "$__object/parameter/check-interval" ]; then echo "When using --check-script you must also define --check-interval" >&2 exit 1 fi -if [ -f "$__object/parameter/check-http" -a ! -f "$__object/parameter/check-interval" ]; then +if [ -f "$__object/parameter/check-http" ] && [ ! -f "$__object/parameter/check-interval" ]; then echo "When using --check-http you must also define --check-interval" >&2 exit 1 fi diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index 5fe657d0..488a0f5d 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -26,11 +26,11 @@ template_dir="/etc/consul-template/template" require="" # Sanity checks -if [ -f "$__object/parameter/source" -a -f "$__object/parameter/source-file" ]; then +if [ -f "$__object/parameter/source" ] && [ -f "$__object/parameter/source-file" ]; then echo "Use either --source OR --source-file, but not both." >&2 exit 1 fi -if [ ! -f "$__object/parameter/source" -a ! -f "$__object/parameter/source-file" ]; then +if [ ! -f "$__object/parameter/source" ] && [ ! -f "$__object/parameter/source-file" ]; then echo "Either --source OR --source-file must be given." >&2 exit 1 fi diff --git a/cdist/conf/type/__consul_watch_checks/manifest b/cdist/conf/type/__consul_watch_checks/manifest index ebb49e2e..146f609e 100755 --- a/cdist/conf/type/__consul_watch_checks/manifest +++ b/cdist/conf/type/__consul_watch_checks/manifest @@ -25,7 +25,7 @@ conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" # Sanity checks -if [ -f "$__object/parameter/filter-service" -a -f "$__object/parameter/filter-state" ]; then +if [ -f "$__object/parameter/filter-service" ] && [ -f "$__object/parameter/filter-state" ]; then echo "Use either --filter-service or --filter-state but not both." >&2 exit 1 fi diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 15a9ee0e..3f60b4b3 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -23,7 +23,7 @@ destination="/$__object_id" state_should="$(cat "$__object/parameter/state")" type="$(cat "$__object/explorer/type")" -[ "$state_should" = "exists" -a "$type" = "file" ] && exit 0 # nothing to do +[ "$state_should" = "exists" ] && [ "$type" = "file" ] && exit 0 # nothing to do if [ "$state_should" = "pre-exists" ]; then if [ -f "$__object/parameter/source" ]; then @@ -41,7 +41,7 @@ fi upload_file= create_file= -if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then +if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ ! -f "$__object/parameter/source" ]; then remote_stat="$(cat "$__object/explorer/stat")" if [ -z "$remote_stat" ]; then @@ -70,7 +70,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then fi fi fi - if [ "$create_file" -o "$upload_file" ]; then + if [ "$create_file" ] || [ "$upload_file" ]; then # tell gencode-remote that we created or uploaded a file and that it must # set all attributes no matter what the explorer retreived mkdir "$__object/files" diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index c90be0be..9dfd1833 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -72,7 +72,7 @@ case "$state_should" in fi value_is="$(get_current_value "$attribute" "$value_should")" - if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then + if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then "set_$attribute" "$value_should" fi fi diff --git a/cdist/conf/type/__git/gencode-remote b/cdist/conf/type/__git/gencode-remote index d0d0d4ed..5a9e23fc 100755 --- a/cdist/conf/type/__git/gencode-remote +++ b/cdist/conf/type/__git/gencode-remote @@ -35,10 +35,10 @@ owner="$(cat "$__object/parameter/owner")" group="$(cat "$__object/parameter/group")" mode="$(cat "$__object/parameter/mode")" -[ "$state_should" = "$state_is" -a \ - "$owner" = "$owner_is" -a \ - "$group" = "$group_is" -a \ - -n "$mode" ] && exit 0 +[ "$state_should" = "$state_is" ] && \ +[ "$owner" = "$owner_is" ] && \ +[ "$group" = "$group_is" ] && \ +[ -n "$mode" ] && exit 0 case $state_should in present) @@ -46,8 +46,8 @@ case $state_should in if [ "$state_should" != "$state_is" ]; then echo git clone --quiet --branch "$branch" "$source" "$destination" fi - if [ \( -n "$owner" -a "$owner_is" != "$owner" \) -o \ - \( -n "$group" -a "$group_is" != "$group" \) ]; then + if { [ -n "$owner" ] && [ "$owner_is" != "$owner" ]; } || \ + { [ -n "$group" ] && [ "$group_is" != "$group" ]; }; then echo chown -R "${owner}:${group}" "$destination" fi if [ -n "$mode" ]; then diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 89bf7b3f..3f784188 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -36,12 +36,12 @@ has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl") # case "$os" in archlinux|debian|suse|ubuntu|devuan|coreos) - if [ "$name_config" = "$name_should" -a "$name_running" = "$name_should" ]; then + if [ "$name_config" = "$name_should" ] && [ "$name_running" = "$name_should" ]; then exit 0 fi ;; scientific|centos|freebsd|openbsd) - if [ "$name_sysconfig" = "$name_should" -a "$name_running" = "$name_should" ]; then + if [ "$name_sysconfig" = "$name_should" ] && [ "$name_running" = "$name_should" ]; then exit 0 fi ;; diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index 090a5d86..bed40ffa 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -127,7 +127,7 @@ for object in $objects; do if [ "${minor}" -lt "5" ]; then # Primary partitions available_device_size=$available_size - if [ "$type" = "extended" -o "$type" = "5" ]; then + if [ "$type" = "extended" ] || [ "$type" = "5" ]; then # Extended partition available_extended_size=$partition_size fi diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 76241e0e..d8bc5eb7 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -36,7 +36,7 @@ state="$(cat "$__object/parameter/state")" started="true" # If the user wants the jail gone, it implies it shouldn't be started. -[ -f "$__object/parameter/stopped" -o "$state" = "absent" ] && started="false" +{ [ -f "$__object/parameter/stopped" ] || [ "$state" = "absent" ]; } && started="false" if [ -f "$__object/parameter/ip" ]; then ip="$(cat "$__object/parameter/ip")" @@ -66,7 +66,7 @@ devfsruleset="$(cat "$__object/parameter/devfs-ruleset")" # devfs_ruleset being defined without devfs_enable being true # is pointless. Treat this as an error. -if [ -n "$devfsruleset" -a "$devfsenable" = "false" ]; then +if [ -n "$devfsruleset" ] && [ "$devfsenable" = "false" ]; then exec >&2 echo "Can't have --devfs-ruleset defined with --devfs-disable" exit 1 diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index 63b48e5c..d883dec4 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -36,7 +36,7 @@ state="$(cat "$__object/parameter/state")" started="true" # If the user wants the jail gone, it implies it shouldn't be started. -[ -f "$__object/parameter/stopped" -o "$state" = "absent" ] && started="false" +{ [ -f "$__object/parameter/stopped" ] || [ "$state" = "absent" ]; } && started="false" if [ -f "$__object/parameter/ip" ]; then ip="$(cat "$__object/parameter/ip")" @@ -70,7 +70,7 @@ devfsruleset="$(cat "$__object/parameter/devfs-ruleset")" # devfs_ruleset being defined without devfs_enable being true # is pointless. Treat this as an error. -if [ -n "$devfsruleset" -a "$devfsenable" = "false" ]; then +if [ -n "$devfsruleset" ] && [ "$devfsenable" = "false" ]; then exec >&2 echo "Can't have --devfs-ruleset defined with --devfs-disable" exit 1 diff --git a/cdist/conf/type/__key_value/manifest b/cdist/conf/type/__key_value/manifest index c7801c89..5a91f60c 100755 --- a/cdist/conf/type/__key_value/manifest +++ b/cdist/conf/type/__key_value/manifest @@ -21,7 +21,7 @@ state_should="$(cat "$__object/parameter/state")" -if [ "$state_should" = "present" -a ! -f "$__object/parameter/value" ]; then +if [ "$state_should" = "present" ] && [ ! -f "$__object/parameter/value" ]; then echo "Missing required parameter 'value'" >&2 exit 1 fi diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 996029f5..044ebe90 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -18,7 +18,7 @@ # along with cdist. If not, see . # -if [ -f "$__object/parameter/before" -a -f "$__object/parameter/after" ]; then +if [ -f "$__object/parameter/before" ] && [ -f "$__object/parameter/after" ]; then echo "Use either --before OR --after but not both." >&2 exit 1 fi diff --git a/cdist/conf/type/__link/gencode-remote b/cdist/conf/type/__link/gencode-remote index 4467fb8e..dc7f3193 100755 --- a/cdist/conf/type/__link/gencode-remote +++ b/cdist/conf/type/__link/gencode-remote @@ -61,7 +61,7 @@ case "$state_should" in ;; absent) # only delete if it is a sym/hard link - if [ "$file_type" = "symlink" -o "$file_type" = "hardlink" ]; then + if [ "$file_type" = "symlink" ] || [ "$file_type" = "hardlink" ]; then printf 'rm -f "%s"\n' "$destination" fi ;; diff --git a/cdist/conf/type/__package/manifest b/cdist/conf/type/__package/manifest index fe7abedc..3ab0f546 100755 --- a/cdist/conf/type/__package/manifest +++ b/cdist/conf/type/__package/manifest @@ -56,7 +56,7 @@ state="$(cat "$__object/parameter/state")" set -- "$@" "$__object_id" "--state" "$state" cd "$__object/parameter" for property in $(ls .); do - if [ "$property" != "type" -a "$property" != "state" ]; then + if [ "$property" != "type" ] && [ "$property" != "state" ]; then set -- "$@" "--$property" "$(cat "$property")" fi done diff --git a/cdist/conf/type/__package_emerge/gencode-remote b/cdist/conf/type/__package_emerge/gencode-remote index 6abe2d61..ada2b414 100755 --- a/cdist/conf/type/__package_emerge/gencode-remote +++ b/cdist/conf/type/__package_emerge/gencode-remote @@ -38,11 +38,11 @@ fi pkg_version="$(cat "$__object/explorer/pkg_version")" if [ -z "$pkg_version" ]; then state_is="absent" -elif [ -z "$version" -a $(echo "$pkg_version" | wc -l) -gt 1 ]; then +elif [ -z "$version" ] && [ $(echo "$pkg_version" | wc -l) -gt 1 ]; then echo "Package name is not unique! The following packages are installed:" echo "$pkg_version" exit 1 -elif [ -n "$version" -a $(echo "$pkg_version" | cut -d " " -f 1 | sort | uniq | wc -l) -gt 1 ]; then +elif [ -n "$version" ] && [ $(echo "$pkg_version" | cut -d " " -f 1 | sort | uniq | wc -l) -gt 1 ]; then echo "Package name is not unique! The following packages are installed:" echo "$pkg_version" exit 1 diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index b51c3153..012bf2ad 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -66,7 +66,7 @@ cmd="" # FIXME: This is ugly. execcmd(){ # Set the PACKAGESITE if we're ADDing a new package - if [ "$1" = "add" -a -n "$pkgsite" ]; then + if [ "$1" = "add" ] && [ -n "$pkgsite" ]; then # Use http.../All/ if we know the exact version we want, use .../Latest/ otherwise pkgsite="export PACKAGESITE=${pkgsite}" [ -n "$version" ] && pkgsite="${pkgsite}/All/" || pkgsite="${pkgsite}/Latest/" diff --git a/cdist/conf/type/__package_pkg_openbsd/gencode-remote b/cdist/conf/type/__package_pkg_openbsd/gencode-remote index 4a6763cd..61383edb 100755 --- a/cdist/conf/type/__package_pkg_openbsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_openbsd/gencode-remote @@ -46,7 +46,7 @@ else name="$__object_id" fi -if [ -n "$version" -a -n "$flavor" ]; then +if [ -n "$version" ] && [ -n "$flavor" ]; then pkgid="$name-$version-$flavor" elif [ -n "$version" ]; then pkgid="$name-$version" diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 2cc76b81..8589c6d1 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -28,7 +28,7 @@ if [ -f "$__object/parameter/maxage" ]; then fi if [ -n "$maxage" ]; then - if [ "$type" != "apt" -a "$type" != "pacman" ]; then + if [ "$type" != "apt" ] && [ "$type" != "pacman" ]; then echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2 exit 1 elif [ $currage -lt $maxage ]; then diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote index a4f078c5..4cba5987 100755 --- a/cdist/conf/type/__pyvenv/gencode-remote +++ b/cdist/conf/type/__pyvenv/gencode-remote @@ -29,10 +29,10 @@ owner="$(cat "$__object/parameter/owner")" group="$(cat "$__object/parameter/group")" mode="$(cat "$__object/parameter/mode")" -[ "$state_should" = "$state_is" -a \ - "$owner" = "$owner_is" -a \ - "$group" = "$group_is" -a \ - -n "$mode" ] && exit 0 +[ "$state_should" = "$state_is" ] && \ +[ "$owner" = "$owner_is" ] && \ +[ "$group" = "$group_is" ] && \ +[ -n "$mode" ] && exit 0 destination="/$__object_id" venvparams="$(cat "$__object/parameter/venvparams")" @@ -49,8 +49,8 @@ case $state_should in if [ "$state_should" != "$state_is" ]; then echo $pyvenv $venvparams "$destination" fi - if [ \( -n "$owner" -a "$owner_is" != "$owner" \) -o \ - \( -n "$group" -a "$group_is" != "$group" \) ]; then + if { [ -n "$owner" ] && [ "$owner_is" != "$owner" ]; } || \ + { [ -n "$group" ] && [ "$group_is" != "$group" ]; }; then echo chown -R "${owner}:${group}" "$destination" fi if [ -n "$mode" ]; then diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest index 9fad8896..8cac3449 100755 --- a/cdist/conf/type/__ssh_authorized_keys/manifest +++ b/cdist/conf/type/__ssh_authorized_keys/manifest @@ -23,7 +23,7 @@ owner="$(cat "$__object/parameter/owner" 2>/dev/null || echo "$__object_id")" state="$(cat "$__object/parameter/state" 2>/dev/null)" file="$(cat "$__object/explorer/file")" -if [ ! -f "$__object/parameter/noparent" -o ! -f "$__object/parameter/nofile" ]; then +if [ ! -f "$__object/parameter/noparent" ] || [ ! -f "$__object/parameter/nofile" ]; then group="$(cut -d':' -f 1 "$__object/explorer/group")" if [ -z "$group" ]; then echo "Failed to get owners group from explorer." >&2 From aef14ad39e9428299cc7dd05305209773ad4fcc6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Oct 2018 17:21:24 +0200 Subject: [PATCH 108/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8fd2ea49..a3d60fbb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) * Core: Add timestamp (optional) to log messages (Darko Poljak) + * Explorers, manifests, gencodes: Fix SC2166: and/or operators in test (shellcheck) (Jonas Weber) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 5ed95ce93b145e41079e9458129f4b82614314bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Oct 2018 20:22:44 +0200 Subject: [PATCH 109/178] Fix SC2005 --- cdist/conf/explorer/cpu_cores | 2 +- cdist/conf/explorer/cpu_sockets | 2 +- cdist/conf/type/__zypper_repo/explorer/all_repo_ids | 2 +- cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids | 2 +- cdist/conf/type/__zypper_repo/explorer/repo_id | 2 +- cdist/conf/type/__zypper_service/explorer/repo_ids | 2 +- cdist/conf/type/__zypper_service/explorer/service_id | 2 +- cdist/conf/type/__zypper_service/explorer/service_ids | 2 +- cdist/conf/type/__zypper_service/explorer/service_uri | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores index 7f7a955e..2f676f7a 100755 --- a/cdist/conf/explorer/cpu_cores +++ b/cdist/conf/explorer/cpu_cores @@ -25,7 +25,7 @@ os=$("$__explorer/os") case "$os" in "macosx") - echo "$(sysctl -n hw.physicalcpu)" + sysctl -n hw.physicalcpu ;; *) diff --git a/cdist/conf/explorer/cpu_sockets b/cdist/conf/explorer/cpu_sockets index 8a8194df..903cc743 100755 --- a/cdist/conf/explorer/cpu_sockets +++ b/cdist/conf/explorer/cpu_sockets @@ -25,7 +25,7 @@ os=$("$__explorer/os") case "$os" in "macosx") - echo "$(system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}')" + system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}' ;; *) diff --git a/cdist/conf/type/__zypper_repo/explorer/all_repo_ids b/cdist/conf/type/__zypper_repo/explorer/all_repo_ids index b37d8ac5..dde6d554 100644 --- a/cdist/conf/type/__zypper_repo/explorer/all_repo_ids +++ b/cdist/conf/type/__zypper_repo/explorer/all_repo_ids @@ -21,4 +21,4 @@ # Retrieve all repo id nummbers - parsed zypper output # # -echo $(zypper lr | cut -d'|' -f 1 | grep -E '^[0-9]') +zypper lr | cut -d'|' -f 1 | grep -E '^[0-9]' diff --git a/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids b/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids index 2dfb946f..b011c258 100644 --- a/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids +++ b/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids @@ -23,4 +23,4 @@ # # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper lr -E | cut -d'|' -f 1 | grep -E '^[0-9]') -echo $(zypper lr | grep -E '^[0-9]([^|]+\|){3,3} Yes' | cut -d'|' -f 1) +zypper lr | grep -E '^[0-9]([^|]+\|){3,3} Yes' | cut -d'|' -f 1 diff --git a/cdist/conf/type/__zypper_repo/explorer/repo_id b/cdist/conf/type/__zypper_repo/explorer/repo_id index 6a4791e6..114c6fe7 100644 --- a/cdist/conf/type/__zypper_repo/explorer/repo_id +++ b/cdist/conf/type/__zypper_repo/explorer/repo_id @@ -26,4 +26,4 @@ if [ -f "$__object/parameter/uri" ]; then else uri="$__object_id" fi -echo $(zypper lr -u | grep -F "$uri" | cut -d'|' -f 1 | grep -E '^[0-9]' ) +zypper lr -u | grep -F "$uri" | cut -d'|' -f 1 | grep -E '^[0-9]' diff --git a/cdist/conf/type/__zypper_service/explorer/repo_ids b/cdist/conf/type/__zypper_service/explorer/repo_ids index e831b76c..787e9869 100644 --- a/cdist/conf/type/__zypper_service/explorer/repo_ids +++ b/cdist/conf/type/__zypper_service/explorer/repo_ids @@ -24,4 +24,4 @@ # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper lr -u -E | cut -d'|' -f 1 | grep -E '^[0-9]') # on older systems, zypper doesn't know the parameter -E -echo $(zypper lr -u | grep -E '^([^|]+\|){3,3} Yes' | cut -d'|' -f 1 | grep -E '^[0-9]') +zypper lr -u | grep -E '^([^|]+\|){3,3} Yes' | cut -d'|' -f 1 | grep -E '^[0-9]' diff --git a/cdist/conf/type/__zypper_service/explorer/service_id b/cdist/conf/type/__zypper_service/explorer/service_id index bf5f0260..91858d84 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_id +++ b/cdist/conf/type/__zypper_service/explorer/service_id @@ -27,4 +27,4 @@ else fi # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper ls -u -E | grep -E "\<$uri\>" | cut -d'|' -f 1 ) -echo $(zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\<$uri\>" | cut -d'|' -f 1 ) +zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\<$uri\>" | cut -d'|' -f 1 diff --git a/cdist/conf/type/__zypper_service/explorer/service_ids b/cdist/conf/type/__zypper_service/explorer/service_ids index 0f1f4186..6491ab90 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_ids +++ b/cdist/conf/type/__zypper_service/explorer/service_ids @@ -22,4 +22,4 @@ # # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper ls -u -E | cut -d'|' -f 1 | grep -E '^[0-9]') -echo $(zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | cut -d'|' -f 1 | grep -E '^[0-9]') +zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | cut -d'|' -f 1 | grep -E '^[0-9]' diff --git a/cdist/conf/type/__zypper_service/explorer/service_uri b/cdist/conf/type/__zypper_service/explorer/service_uri index 6eee47fb..b8de0dcd 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_uri +++ b/cdist/conf/type/__zypper_service/explorer/service_uri @@ -25,4 +25,4 @@ if [ -f "$__object/parameter/uri" ]; then else uri="/$__object_id" fi -echo $(zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'$uri'" {print $NF}') +zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'$uri'" {print $NF}' From 0928708d19b77a1ca0ad7f0b3f7c0d74c81bfa9a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Oct 2018 20:34:54 +0200 Subject: [PATCH 110/178] Fix SC2004 --- .../__install_partition_msdos_apply/gencode-remote | 10 +++++----- cdist/conf/type/__prometheus_alertmanager/manifest | 2 +- cdist/conf/type/__prometheus_server/manifest | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index 090a5d86..7e8eef4e 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -40,16 +40,16 @@ size_to_mb() { case "$suffix" in K|k) - size="$(( $number / 1024 ))" + size="$(( number / 1024 ))" ;; M|m) size="$number" ;; G|g) - size="$(( $number * 1024 ))" + size="$(( number * 1024 ))" ;; %) - size="$(( $available_size * $number / 100 ))" + size="$(( available_size * number / 100 ))" ;; *) size="-1" @@ -108,7 +108,7 @@ for object in $objects; do if [ "${minor}" -lt "5" ]; then # Primary partitions - primary_count=$(( $primary_count + 1 )) + primary_count=$(( primary_count + 1 )) available_size=$available_device_size else # Logical partitions @@ -121,7 +121,7 @@ for object in $objects; do available_size=0 else partition_size=$(size_to_mb "$size" "$available_size") - available_size="$(( $available_size - $partition_size ))" + available_size="$(( available_size - partition_size ))" fi if [ "${minor}" -lt "5" ]; then diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index be50b71e..054e44cb 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -48,7 +48,7 @@ __key_value alertmanager_fix_init_script --file /etc/init.d/prometheus-alertmana ##### CONFIGURE ############################################################# -FLAGS="--storage.path $storage_path --data.retention $(($retention_days*24))h --web.listen-address [::]:9093" +FLAGS="--storage.path $storage_path --data.retention $((retention_days*24))h --web.listen-address [::]:9093" require="$require $require_pkg" \ __key_value alertmanager_args --file /etc/default/prometheus-alertmanager \ diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index 84ba53cf..e2f32fd5 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -45,7 +45,7 @@ require="$require $require_pkg" __directory "$storage_path" --owner prometheus - ##### CONFIGURE ############################################################# -FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $(($retention_days*24))h --web.listen-address [::]:9090" +FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $((retention_days*24))h --web.listen-address [::]:9090" # TODO it would be neat to restart prometheus on change -- __key_value really should have an --onchange parameter require="$require $require_pkg" \ From 3757e8c2c504367bdd2d6b69a5aafbbf0fdba772 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Oct 2018 21:50:17 +0200 Subject: [PATCH 111/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index a3d60fbb..e51c198c 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,8 @@ next: * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) * Core: Add timestamp (optional) to log messages (Darko Poljak) * Explorers, manifests, gencodes: Fix SC2166: and/or operators in test (shellcheck) (Jonas Weber) + * Explorers and types: Fix SC2004: $/${} is unnecessary on arithmetic variables (shellcheck) (Darko Poljak) + * Explorers and types: Fix SC2005: Useless echo? Instead of echo $(cmd), just use cmd (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 44acfcdd12a8e1d8cfaa91c38c87e277c79f819d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 2 Oct 2018 20:44:46 +0200 Subject: [PATCH 112/178] Fix SC2002 --- cdist/conf/explorer/cpu_sockets | 4 ++-- cdist/conf/explorer/machine_type | 2 +- cdist/conf/type/__ccollect_source/manifest | 2 +- cdist/conf/type/__consul/gencode-remote | 2 +- cdist/conf/type/__golang_from_vendor/gencode-remote | 2 +- cdist/conf/type/__ssh_authorized_key/explorer/entry | 2 +- cdist/conf/type/__ssh_authorized_key/gencode-remote | 2 +- cdist/conf/type/__staged_file/gencode-local | 2 +- cdist/conf/type/__zypper_service/gencode-remote | 2 +- cdist/conf/type/__zypper_service/manifest | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cdist/conf/explorer/cpu_sockets b/cdist/conf/explorer/cpu_sockets index 903cc743..2d577043 100755 --- a/cdist/conf/explorer/cpu_sockets +++ b/cdist/conf/explorer/cpu_sockets @@ -30,9 +30,9 @@ case "$os" in *) if [ -r /proc/cpuinfo ]; then - sockets="$(grep "physical id" /proc/cpuinfo | sort | uniq | wc -l)" + sockets="$(grep "physical id" /proc/cpuinfo | sort -u | wc -l)" if [ ${sockets} -eq 0 ]; then - sockets="$(cat /proc/cpuinfo | grep "processor" | wc -l)" + sockets="$(grep -c "processor" /proc/cpuinfo)" fi echo "${sockets}" fi diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 3b4f0308..3a3cd0c1 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -28,7 +28,7 @@ if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then fi if [ -e "/proc/1/environ" ] && - cat "/proc/1/environ" | tr '\000' '\n' | grep -Eiq '^container='; then + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then echo lxc exit fi diff --git a/cdist/conf/type/__ccollect_source/manifest b/cdist/conf/type/__ccollect_source/manifest index 238c7e76..26c6cc99 100755 --- a/cdist/conf/type/__ccollect_source/manifest +++ b/cdist/conf/type/__ccollect_source/manifest @@ -22,7 +22,7 @@ name="$__object_id" state="$(cat "$__object/parameter/state")" source="$(cat "$__object/parameter/source")" destination="$(cat "$__object/parameter/destination")" -ccollectconf="$(cat "$__object/parameter/ccollectconf" | sed 's,/$,,')" +ccollectconf="$(sed 's,/$,,' "$__object/parameter/ccollectconf")" sourcedir="$ccollectconf/sources" basedir="$sourcedir/$name" diff --git a/cdist/conf/type/__consul/gencode-remote b/cdist/conf/type/__consul/gencode-remote index 22e9eea1..1d2244ea 100755 --- a/cdist/conf/type/__consul/gencode-remote +++ b/cdist/conf/type/__consul/gencode-remote @@ -39,7 +39,7 @@ version_dir="$versions_dir/$version" source=$(cat "$version_dir/source") source_file_name="${source##*/}" -cksum_should=$(cat "$version_dir/cksum" | cut -d' ' -f1,2) +cksum_should=$(cut -d' ' -f1,2 "$version_dir/cksum") cat << eof tmpdir=\$(mktemp -d --tmpdir="/tmp" "${__type##*/}.XXXXXXXXXX") diff --git a/cdist/conf/type/__golang_from_vendor/gencode-remote b/cdist/conf/type/__golang_from_vendor/gencode-remote index 1654978b..5200e9e3 100755 --- a/cdist/conf/type/__golang_from_vendor/gencode-remote +++ b/cdist/conf/type/__golang_from_vendor/gencode-remote @@ -2,7 +2,7 @@ version=$(cat "$__object/parameter/version") -kernel_name=$(cat "$__global/explorer/kernel_name" | tr '[:upper:]' '[:lower:]') +kernel_name=$(tr '[:upper:]' '[:lower:]' < "$__global/explorer/kernel_name") machine=$(cat "$__global/explorer/machine") case $machine in x86_64|amd64) diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index 1535d348..157d70f1 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -19,7 +19,7 @@ # # extract the keytype and base64 encoded key ignoring any options and comment -type_and_key="$(cat "$__object/parameter/key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" +type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" # If type_and_key is empty, which is the case with an invalid key, do not grep $file because it results # in greping everything in file and all entries from file are removed. if [ -n "${type_and_key}" ] diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 7ded7dc6..333dfa03 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -59,7 +59,7 @@ mkdir "$__object/files" ( if [ -f "$__object/parameter/option" ]; then # comma seperated list of options - options="$(cat "$__object/parameter/option" | tr '\n' ',')" + options="$(tr '\n' ',' < "$__object/parameter/option")" printf '%s ' "${options%*,}" fi if [ -f "$__object/parameter/comment" ]; then diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local index 8e2003af..18bf09f5 100755 --- a/cdist/conf/type/__staged_file/gencode-local +++ b/cdist/conf/type/__staged_file/gencode-local @@ -74,7 +74,7 @@ fetch_and_prepare_file() { cat << DONE verify_cksum() { cksum_is="\$(cksum "$stage_file" | cut -d' ' -f1,2)" - cksum_should="$(cat "$__object/parameter/cksum" | cut -d' ' -f1,2)" + cksum_should="$(cut -d' ' -f1,2 "$__object/parameter/cksum")" if [ "\$cksum_is" = "\$cksum_should" ]; then return 0 else diff --git a/cdist/conf/type/__zypper_service/gencode-remote b/cdist/conf/type/__zypper_service/gencode-remote index e5b41cc6..955698d1 100755 --- a/cdist/conf/type/__zypper_service/gencode-remote +++ b/cdist/conf/type/__zypper_service/gencode-remote @@ -46,7 +46,7 @@ exp_uri="$(cat "$__object/explorer/service_uri")" exp_id="$(cat "$__object/explorer/service_id")" # we need this list to remove ids, but we must do this in reverse order -exp_ids="$(cat "$__object/explorer/service_ids" | rev)" +exp_ids="$(rev "$__object/explorer/service_ids")" if [ "$uri" = "$exp_uri" ] ; then state_is="present" diff --git a/cdist/conf/type/__zypper_service/manifest b/cdist/conf/type/__zypper_service/manifest index e4f0bcf6..42a56830 100755 --- a/cdist/conf/type/__zypper_service/manifest +++ b/cdist/conf/type/__zypper_service/manifest @@ -47,7 +47,7 @@ fi [ "$state_is" = "$state_should" ] && exit 0 # we need this list to remove ids, but we must do this in reverse order -exp_repos="$(cat "$__object/explorer/repo_ids" | rev)" +exp_repos="$(rev "$__object/explorer/repo_ids")" # boolean parameter if [ -f "$__object/parameter/remove-all-repos" ]; then From 8ef2773c0db54179ccb456961758aaea53aef2ee Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 3 Oct 2018 14:32:28 +0200 Subject: [PATCH 113/178] Fix SC2039 --- cdist/conf/type/__docker/manifest | 2 +- cdist/conf/type/__group/gencode-remote | 6 +++--- .../files/lib.sh | 20 +++++++++---------- .../gencode-remote | 10 +++++----- .../explorer/pkg_version | 2 +- .../explorer/pkg_version | 2 +- .../__package_pkgng_freebsd/gencode-remote | 2 +- 7 files changed, 22 insertions(+), 22 deletions(-) diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index d501a9f1..e0558b46 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -24,7 +24,7 @@ state=$(cat "$__object/parameter/state") case "$os" in centos) - if (source "$__global/explorer/os_release" && [ "${VERSION_ID}" = "7" ]); then + if (. "$__global/explorer/os_release" && [ "${VERSION_ID}" = "7" ]); then __yum_repo docker-ce-stable \ --name 'Docker CE Stable' \ --baseurl "https://download.docker.com/linux/centos/7/\$basearch/stable" \ diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote index 5847cb66..68475178 100755 --- a/cdist/conf/type/__group/gencode-remote +++ b/cdist/conf/type/__group/gencode-remote @@ -30,9 +30,9 @@ state="$(cat "$__object/parameter/state")" # Use short option names for portability shorten_property() { case "$1" in - gid) echo "-g";; - password) echo "-p";; - system) echo "-r";; + gid) echo -- "-g";; + password) echo -- "-p";; + system) echo -- "-r";; esac } diff --git a/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh b/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh index cddc575d..13ead401 100644 --- a/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh +++ b/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh @@ -8,8 +8,8 @@ debug() { } fdisk_command() { - local device="$1" - local cmd="$2" + device="$1" + cmd="$2" debug fdisk_command "running fdisk command '${cmd}' on device ${device}" printf "${cmd}\nw\n" | fdisk -c -u "$device" @@ -20,7 +20,7 @@ fdisk_command() { } create_disklabel() { - local device=$1 + device=$1 debug create_disklabel "creating new msdos disklabel" fdisk_command ${device} "o" @@ -28,18 +28,18 @@ create_disklabel() { } toggle_bootable() { - local device="$1" - local minor="$2" + device="$1" + minor="$2" fdisk_command ${device} "a\n${minor}\n" return $? } create_partition() { - local device="$1" - local minor="$2" - local size="$3" - local type="$4" - local primary_count="$5" + device="$1" + minor="$2" + size="$3" + type="$4" + primary_count="$5" if [ "$type" = "extended" -o "$type" = "5" ]; then # Extended partition diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index 312e6f81..9e5fcd00 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -31,12 +31,12 @@ debug() { # Convert a size specifier 1G 100M or 50% into the corresponding numeric MB. size_to_mb() { - local size=$1 - local available_size="$2" + size=$1 + available_size="$2" - local number_suffix="$(echo ${size} | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" - local number="$(echo ${number_suffix} | cut -d '|' -f1)" - local suffix="$(echo ${number_suffix} | cut -d '|' -f2)" + number_suffix="$(echo ${size} | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" + number="$(echo ${number_suffix} | cut -d '|' -f1)" + suffix="$(echo ${number_suffix} | cut -d '|' -f2)" case "$suffix" in K|k) diff --git a/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version index 1335ba79..0a1ab75c 100755 --- a/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version +++ b/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version @@ -30,7 +30,7 @@ fi # Don't produce "no pkgs installed" output -- breaks things PKG_OUTPUT=$(pkg_info 2>&1) if [ ! "$PKG_OUTPUT" = "pkg_info: no packages installed" ]; then - echo -n "$(echo "$PKG_OUTPUT" \ + printf "%s" "$(echo "$PKG_OUTPUT" \ | awk '{print $1}' \ | sed 's/^\(.*\)-\([^-]*\)$/name:\1 ver:\2/g' \ | grep "name:$name ver:" \ diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version index 947857b9..92ce0623 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version +++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version @@ -29,7 +29,7 @@ fi # Don't produce "no pkgs installed" output -- breaks things PKG_OUTPUT=$(pkg info 2>&1) -echo -n "$(echo "$PKG_OUTPUT" \ +printf "%s" "$(echo "$PKG_OUTPUT" \ | awk '{print $1}' \ | sed 's/^\(.*\)-\([^-]*\)$/name:\1 ver:\2/g' \ | grep "name:$name ver:" \ diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index d21e9e2a..b59e3648 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -52,7 +52,7 @@ cmd="" # Parms: $1 -- mode, "rm", "add", or "upg" # $2 -- the command to be echoed execcmd(){ - local _cmd="" + _cmd="" case "$1" in add) From 383af6736f968cd79d742b300fffaf97a0c82713 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 3 Oct 2018 14:52:17 +0200 Subject: [PATCH 114/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e51c198c..ecd52327 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorers, manifests, gencodes: Fix SC2166: and/or operators in test (shellcheck) (Jonas Weber) * Explorers and types: Fix SC2004: $/${} is unnecessary on arithmetic variables (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2005: Useless echo? Instead of echo $(cmd), just use cmd (shellcheck) (Darko Poljak) + * Explorers and types: Fix SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From ab9e2264df73c9f97db502ab4efa7701a2c90371 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 09:10:31 +0200 Subject: [PATCH 115/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ecd52327..100b914d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Explorers and types: Fix SC2004: $/${} is unnecessary on arithmetic variables (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2005: Useless echo? Instead of echo $(cmd), just use cmd (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead (shellcheck) (Darko Poljak) + * Explorers and types: Fix SC2039: In POSIX sh, something is undefined (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 134c84607f0ae90666ca17d7454462cb111fde90 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 4 Oct 2018 12:28:59 +0200 Subject: [PATCH 116/178] fix another SC2166 (test `[ .. -a.. ]` replaced by `[ .. ] && ]`) --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 3a3cd0c1..bb21f69c 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -22,7 +22,7 @@ # FIXME: other system types (not linux ...) -if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then +if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then echo openvz exit fi From 31bf6ab230f68b55bd53b7646c3abd3487fc7303 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 12:38:40 +0200 Subject: [PATCH 117/178] ++changelog --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 100b914d..5fccbc55 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,7 +4,7 @@ Changelog next: * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) * Core: Add timestamp (optional) to log messages (Darko Poljak) - * Explorers, manifests, gencodes: Fix SC2166: and/or operators in test (shellcheck) (Jonas Weber) + * Explorers, manifests, gencodes: Fix SC2166: and/or operators in test (shellcheck) (Jonas Weber, Thomas Eckert) * Explorers and types: Fix SC2004: $/${} is unnecessary on arithmetic variables (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2005: Useless echo? Instead of echo $(cmd), just use cmd (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead (shellcheck) (Darko Poljak) From d950ddada3a43c8c3d1498a533c91abb6604b51e Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 4 Oct 2018 16:01:45 +0200 Subject: [PATCH 118/178] fix SC2045 (use globs instead of `ls`) --- cdist/conf/type/__config_file/manifest | 3 ++- cdist/conf/type/__consul_agent/manifest | 3 ++- cdist/conf/type/__consul_check/manifest | 3 ++- cdist/conf/type/__consul_service/manifest | 3 ++- cdist/conf/type/__consul_template/manifest | 3 ++- cdist/conf/type/__consul_template_template/manifest | 3 ++- cdist/conf/type/__consul_watch_checks/manifest | 3 ++- cdist/conf/type/__consul_watch_event/manifest | 3 ++- cdist/conf/type/__consul_watch_key/manifest | 3 ++- cdist/conf/type/__consul_watch_keyprefix/manifest | 3 ++- cdist/conf/type/__consul_watch_nodes/manifest | 3 ++- cdist/conf/type/__consul_watch_service/manifest | 3 ++- cdist/conf/type/__consul_watch_services/manifest | 3 ++- cdist/conf/type/__jail/manifest | 2 +- cdist/conf/type/__package/manifest | 2 +- cdist/conf/type/__user/gencode-remote | 4 ++-- 16 files changed, 30 insertions(+), 17 deletions(-) diff --git a/cdist/conf/type/__config_file/manifest b/cdist/conf/type/__config_file/manifest index 3155f79b..be8f9f67 100755 --- a/cdist/conf/type/__config_file/manifest +++ b/cdist/conf/type/__config_file/manifest @@ -19,7 +19,8 @@ # set -- "/${__object_id}" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in source) source="$(cat "$__object/parameter/source")" diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index a696894b..3951f728 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -84,7 +84,8 @@ echo "{" # parameters we define ourself printf ' "data_dir": "%s"\n' "$data_dir" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state|user|group|json-config) continue ;; ca-file-source|cert-file-source|key-file-source) diff --git a/cdist/conf/type/__consul_check/manifest b/cdist/conf/type/__consul_check/manifest index 554c0680..c9f7add9 100755 --- a/cdist/conf/type/__consul_check/manifest +++ b/cdist/conf/type/__consul_check/manifest @@ -50,7 +50,8 @@ fi echo "{" printf ' "check": {\n' printf ' "name": "%s"\n' "$name" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state|name) continue ;; *) diff --git a/cdist/conf/type/__consul_service/manifest b/cdist/conf/type/__consul_service/manifest index f6d29b4a..60397db7 100755 --- a/cdist/conf/type/__consul_service/manifest +++ b/cdist/conf/type/__consul_service/manifest @@ -42,7 +42,8 @@ fi echo "{" printf ' "service": {\n' printf ' "name": "%s"\n' "$name" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state|name|check-interval) continue ;; check-script) diff --git a/cdist/conf/type/__consul_template/manifest b/cdist/conf/type/__consul_template/manifest index 2236e5bd..b02fc332 100755 --- a/cdist/conf/type/__consul_template/manifest +++ b/cdist/conf/type/__consul_template/manifest @@ -75,7 +75,8 @@ require="__directory/etc/consul-template" \ # Generate hcl config file ( -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in auth-password|state|ssl-*|syslog-*|version|vault-token|vault-ssl*) continue ;; auth-username) diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index 488a0f5d..7834ae9c 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -38,7 +38,8 @@ fi # Generate hcl config file ( printf 'template {\n' -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in source-file) source="$(cat "$__object/parameter/$param")" diff --git a/cdist/conf/type/__consul_watch_checks/manifest b/cdist/conf/type/__consul_watch_checks/manifest index 146f609e..5fdd7a74 100755 --- a/cdist/conf/type/__consul_watch_checks/manifest +++ b/cdist/conf/type/__consul_watch_checks/manifest @@ -35,7 +35,8 @@ fi echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; filter-*) diff --git a/cdist/conf/type/__consul_watch_event/manifest b/cdist/conf/type/__consul_watch_event/manifest index 099054a5..61934656 100755 --- a/cdist/conf/type/__consul_watch_event/manifest +++ b/cdist/conf/type/__consul_watch_event/manifest @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_key/manifest b/cdist/conf/type/__consul_watch_key/manifest index 099054a5..61934656 100755 --- a/cdist/conf/type/__consul_watch_key/manifest +++ b/cdist/conf/type/__consul_watch_key/manifest @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_keyprefix/manifest b/cdist/conf/type/__consul_watch_keyprefix/manifest index 099054a5..61934656 100755 --- a/cdist/conf/type/__consul_watch_keyprefix/manifest +++ b/cdist/conf/type/__consul_watch_keyprefix/manifest @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_nodes/manifest b/cdist/conf/type/__consul_watch_nodes/manifest index 099054a5..61934656 100755 --- a/cdist/conf/type/__consul_watch_nodes/manifest +++ b/cdist/conf/type/__consul_watch_nodes/manifest @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_service/manifest b/cdist/conf/type/__consul_watch_service/manifest index 2825c716..db38eb18 100755 --- a/cdist/conf/type/__consul_watch_service/manifest +++ b/cdist/conf/type/__consul_watch_service/manifest @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; passingonly) diff --git a/cdist/conf/type/__consul_watch_services/manifest b/cdist/conf/type/__consul_watch_services/manifest index 099054a5..61934656 100755 --- a/cdist/conf/type/__consul_watch_services/manifest +++ b/cdist/conf/type/__consul_watch_services/manifest @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__jail/manifest b/cdist/conf/type/__jail/manifest index c3d9dfbe..7564be26 100755 --- a/cdist/conf/type/__jail/manifest +++ b/cdist/conf/type/__jail/manifest @@ -39,7 +39,7 @@ __directory ${jaildir} --parents set -- "$@" "$__object_id" "--state" "$state" cd "$__object/parameter" -for property in $(ls .); do +for property in *; do set -- "$@" "--$property" "$(cat "$property")" done diff --git a/cdist/conf/type/__package/manifest b/cdist/conf/type/__package/manifest index 3ab0f546..f9de1145 100755 --- a/cdist/conf/type/__package/manifest +++ b/cdist/conf/type/__package/manifest @@ -55,7 +55,7 @@ state="$(cat "$__object/parameter/state")" set -- "$@" "$__object_id" "--state" "$state" cd "$__object/parameter" -for property in $(ls .); do +for property in *; do if [ "$property" != "type" ] && [ "$property" != "state" ]; then set -- "$@" "--$property" "$(cat "$property")" fi diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index 23762065..332e93f1 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -52,7 +52,7 @@ shorten_property() { if [ "$state" = "present" ]; then cd "$__object/parameter" if grep -q "^${name}:" "$__object/explorer/passwd"; then - for property in $(ls .); do + for property in *; do new_value="$(cat "$property")" unset current_value @@ -113,7 +113,7 @@ if [ "$state" = "present" ]; then fi else echo add >> "$__messages_out" - for property in $(ls .); do + for property in *; do [ "$property" = "state" ] && continue [ "$property" = "remove-home" ] && continue new_value="$(cat "$property")" From d49d38481fb996e47c51f50fad60097c9068c067 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 17:31:37 +0200 Subject: [PATCH 119/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5fccbc55..47639dd5 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Explorers and types: Fix SC2005: Useless echo? Instead of echo $(cmd), just use cmd (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2039: In POSIX sh, something is undefined (shellcheck) (Darko Poljak) + * Explorers and types: Fix SC2045: Iterating over ls output is fragile. Use globs. (shellcheck) (Thomas Eckert) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From a46da35bbcb9aef6e0106e764f2a3332048896d6 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 4 Oct 2018 12:20:50 +0200 Subject: [PATCH 120/178] fix SC2148 (missing shebang) and make executable --- cdist/conf/explorer/disks | 1 + cdist/conf/explorer/is-freebsd-jail | 1 + cdist/conf/explorer/kernel_name | 1 + cdist/conf/type/__daemontools_service/explorer/svc | 1 + cdist/conf/type/__go_get/explorer/go-executable | 1 + 5 files changed, 5 insertions(+) mode change 100644 => 100755 cdist/conf/explorer/disks mode change 100644 => 100755 cdist/conf/explorer/is-freebsd-jail mode change 100644 => 100755 cdist/conf/explorer/kernel_name mode change 100644 => 100755 cdist/conf/type/__daemontools_service/explorer/svc mode change 100644 => 100755 cdist/conf/type/__go_get/explorer/go-executable diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks old mode 100644 new mode 100755 index 52fef81e..6febdbb4 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -1,2 +1,3 @@ +#!/bin/sh -e cd /dev echo sd? hd? vd? diff --git a/cdist/conf/explorer/is-freebsd-jail b/cdist/conf/explorer/is-freebsd-jail old mode 100644 new mode 100755 index a6d11d1a..d8f8b0ac --- a/cdist/conf/explorer/is-freebsd-jail +++ b/cdist/conf/explorer/is-freebsd-jail @@ -1 +1,2 @@ +#!/bin/sh -e sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true diff --git a/cdist/conf/explorer/kernel_name b/cdist/conf/explorer/kernel_name old mode 100644 new mode 100755 index 98ebac2a..365b5029 --- a/cdist/conf/explorer/kernel_name +++ b/cdist/conf/explorer/kernel_name @@ -1 +1,2 @@ +#!/bin/sh -e uname -s diff --git a/cdist/conf/type/__daemontools_service/explorer/svc b/cdist/conf/type/__daemontools_service/explorer/svc old mode 100644 new mode 100755 index d33fcea4..fdca5e34 --- a/cdist/conf/type/__daemontools_service/explorer/svc +++ b/cdist/conf/type/__daemontools_service/explorer/svc @@ -1 +1,2 @@ +#!/bin/sh -e command -v svc || true diff --git a/cdist/conf/type/__go_get/explorer/go-executable b/cdist/conf/type/__go_get/explorer/go-executable old mode 100644 new mode 100755 index 4c84ce07..b1dc0984 --- a/cdist/conf/type/__go_get/explorer/go-executable +++ b/cdist/conf/type/__go_get/explorer/go-executable @@ -1,3 +1,4 @@ +#!/bin/sh -e [ -f /etc/environment ] && . /etc/environment [ -f /etc/profile ] && . /etc/profile go version 2>/dev/null || true From 085b2a2b104cdeeac8200e5d991137f248244a44 Mon Sep 17 00:00:00 2001 From: Thomas Eckert Date: Thu, 4 Oct 2018 15:15:57 +0200 Subject: [PATCH 121/178] remove `set -e` on explorers (they should be able to fail gracefully) --- cdist/conf/explorer/disks | 2 +- cdist/conf/explorer/is-freebsd-jail | 2 +- cdist/conf/explorer/kernel_name | 2 +- cdist/conf/type/__daemontools_service/explorer/svc | 2 +- cdist/conf/type/__go_get/explorer/go-executable | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index 6febdbb4..7a5c0da0 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -1,3 +1,3 @@ -#!/bin/sh -e +#!/bin/sh cd /dev echo sd? hd? vd? diff --git a/cdist/conf/explorer/is-freebsd-jail b/cdist/conf/explorer/is-freebsd-jail index d8f8b0ac..010917f5 100755 --- a/cdist/conf/explorer/is-freebsd-jail +++ b/cdist/conf/explorer/is-freebsd-jail @@ -1,2 +1,2 @@ -#!/bin/sh -e +#!/bin/sh sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true diff --git a/cdist/conf/explorer/kernel_name b/cdist/conf/explorer/kernel_name index 365b5029..1f9cfca4 100755 --- a/cdist/conf/explorer/kernel_name +++ b/cdist/conf/explorer/kernel_name @@ -1,2 +1,2 @@ -#!/bin/sh -e +#!/bin/sh uname -s diff --git a/cdist/conf/type/__daemontools_service/explorer/svc b/cdist/conf/type/__daemontools_service/explorer/svc index fdca5e34..9ba462f2 100755 --- a/cdist/conf/type/__daemontools_service/explorer/svc +++ b/cdist/conf/type/__daemontools_service/explorer/svc @@ -1,2 +1,2 @@ -#!/bin/sh -e +#!/bin/sh command -v svc || true diff --git a/cdist/conf/type/__go_get/explorer/go-executable b/cdist/conf/type/__go_get/explorer/go-executable index b1dc0984..bdce7559 100755 --- a/cdist/conf/type/__go_get/explorer/go-executable +++ b/cdist/conf/type/__go_get/explorer/go-executable @@ -1,4 +1,4 @@ -#!/bin/sh -e +#!/bin/sh [ -f /etc/environment ] && . /etc/environment [ -f /etc/profile ] && . /etc/profile go version 2>/dev/null || true From ef8ec8641e4959f6536f6f5d5caa417954195468 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 18:47:28 +0200 Subject: [PATCH 122/178] ++ --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 47639dd5..1405ad3c 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,7 @@ next: * Explorers and types: Fix SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2039: In POSIX sh, something is undefined (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2045: Iterating over ls output is fragile. Use globs. (shellcheck) (Thomas Eckert) + * Explorers and types: Fix SC2148: Tips depend on target shell and yours is unknown. Add a shebang. (shellcheck) (Thomas Eckert) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 6b0f8fba22a56e853b97706813e59e98963bc7a5 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Wed, 3 Oct 2018 21:11:59 +0200 Subject: [PATCH 123/178] Remove unused variables (almost, found bugs?) Solves SC2034 references #540. --- cdist/conf/type/__apt_ppa/manifest | 2 -- cdist/conf/type/__block/manifest | 2 -- cdist/conf/type/__ccollect_source/gencode-remote | 1 - cdist/conf/type/__file/gencode-remote | 1 - cdist/conf/type/__firewalld_rule/gencode-remote | 1 - cdist/conf/type/__group/gencode-remote | 2 -- cdist/conf/type/__install_generate_fstab/gencode-local | 1 - cdist/conf/type/__install_mkfs/manifest | 8 +------- cdist/conf/type/__install_reboot/gencode-remote | 2 -- cdist/conf/type/__install_reboot/manifest | 6 ++++-- cdist/conf/type/__install_umount/manifest | 6 ++++-- cdist/conf/type/__jail_freebsd10/gencode-remote | 1 + cdist/conf/type/__jail_freebsd9/gencode-remote | 1 + cdist/conf/type/__pacman_conf_integrate/manifest | 2 -- cdist/conf/type/__postfix_master/manifest | 1 - cdist/conf/type/__qemu_img/manifest | 1 - cdist/conf/type/__rvm_gem/gencode-remote | 2 -- cdist/conf/type/__rvm_gemset/explorer/state | 3 --- cdist/conf/type/__rvm_ruby/gencode-remote | 1 - cdist/conf/type/__staged_file/gencode-local | 1 - cdist/conf/type/__staged_file/manifest | 4 ---- 21 files changed, 11 insertions(+), 38 deletions(-) diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index e1af21bd..c6f4e876 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -18,8 +18,6 @@ # along with cdist. If not, see . # -name="$__object_id" - __package software-properties-common require="__package/software-properties-common" \ diff --git a/cdist/conf/type/__block/manifest b/cdist/conf/type/__block/manifest index 8fea3e83..726950d3 100755 --- a/cdist/conf/type/__block/manifest +++ b/cdist/conf/type/__block/manifest @@ -18,8 +18,6 @@ # along with cdist. If not, see . # - -file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")" prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id") suffix=$(cat "$__object/parameter/suffix" 2>/dev/null || echo "#/cdist:__block/$__object_id") text=$(cat "$__object/parameter/text") diff --git a/cdist/conf/type/__ccollect_source/gencode-remote b/cdist/conf/type/__ccollect_source/gencode-remote index 763f219e..c8892c9e 100755 --- a/cdist/conf/type/__ccollect_source/gencode-remote +++ b/cdist/conf/type/__ccollect_source/gencode-remote @@ -56,7 +56,6 @@ set_mode() { echo chmod $1 >> "$__messages_out" } -set_attributes= case "$state_should" in present|exists) # Note: Mode - needs to happen last as a chown/chgrp can alter mode by diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 9dfd1833..6632935c 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -57,7 +57,6 @@ set_mode() { echo chmod $1 >> "$__messages_out" } -set_attributes= case "$state_should" in present|exists|pre-exists) # Note: Mode - needs to happen last as a chown/chgrp can alter mode by diff --git a/cdist/conf/type/__firewalld_rule/gencode-remote b/cdist/conf/type/__firewalld_rule/gencode-remote index 4c824d39..b9b930e7 100755 --- a/cdist/conf/type/__firewalld_rule/gencode-remote +++ b/cdist/conf/type/__firewalld_rule/gencode-remote @@ -19,7 +19,6 @@ # # -name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/rule")" diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote index 68475178..033228c5 100755 --- a/cdist/conf/type/__group/gencode-remote +++ b/cdist/conf/type/__group/gencode-remote @@ -40,11 +40,9 @@ shorten_property() { if [ "$state" = "present" ]; then case "$os" in freebsd) - supported_add_properties="gid" supported_change_properties="gid" ;; *) - supported_add_properties="gid password system" supported_change_properties="gid password" ;; esac diff --git a/cdist/conf/type/__install_generate_fstab/gencode-local b/cdist/conf/type/__install_generate_fstab/gencode-local index 5cc7d877..b5158a39 100755 --- a/cdist/conf/type/__install_generate_fstab/gencode-local +++ b/cdist/conf/type/__install_generate_fstab/gencode-local @@ -28,7 +28,6 @@ $__remote_exec $__target_host blkid > "$__object/files/blkid" for object in $(find "$__global/object/__install_mount" -type d -name "$__cdist_object_marker"); do device="$(cat "$object/parameter/device")" dir="$(cat "$object/parameter/dir")" - prefix="$(cat "$object/parameter/prefix")" type="$(cat "$object/parameter/type")" if [ -f "$object/parameter/options" ]; then options="$(cat "$object/parameter/options")" diff --git a/cdist/conf/type/__install_mkfs/manifest b/cdist/conf/type/__install_mkfs/manifest index eb65757f..b0a21dae 100755 --- a/cdist/conf/type/__install_mkfs/manifest +++ b/cdist/conf/type/__install_mkfs/manifest @@ -19,13 +19,7 @@ # # set defaults -if [ -f "$__object/parameter/device" ]; then - device="(cat "$__object/parameter/device")" -else +if [ ! -f "$__object/parameter/device" ]; then device="/$__object_id" echo "$device" > "$__object/parameter/device" fi - -type="(cat "$__object/parameter/type")" - -options="(cat "$__object/parameter/options")" diff --git a/cdist/conf/type/__install_reboot/gencode-remote b/cdist/conf/type/__install_reboot/gencode-remote index 00c04523..9a6322c1 100755 --- a/cdist/conf/type/__install_reboot/gencode-remote +++ b/cdist/conf/type/__install_reboot/gencode-remote @@ -18,8 +18,6 @@ # along with cdist. If not, see . # -options="$(cat "$__object/parameter/options")" - #echo "reboot $options" cat << DONE echo 1 > /proc/sys/kernel/sysrq diff --git a/cdist/conf/type/__install_reboot/manifest b/cdist/conf/type/__install_reboot/manifest index 02689d82..46a6356e 100755 --- a/cdist/conf/type/__install_reboot/manifest +++ b/cdist/conf/type/__install_reboot/manifest @@ -19,5 +19,7 @@ # # set defaults -options="$(cat "$__object/parameter/options" 2>/dev/null \ - || echo "" | tee "$__object/parameter/options")" +# TODO is this neccesary or should this be handled using the usual parameter/default workflow? +if [ ! -f "$__object/parameter/options" ]; then + touch "$__object/parameter/options" +fi diff --git a/cdist/conf/type/__install_umount/manifest b/cdist/conf/type/__install_umount/manifest index 42cd19bf..d900df1b 100755 --- a/cdist/conf/type/__install_umount/manifest +++ b/cdist/conf/type/__install_umount/manifest @@ -19,5 +19,7 @@ # # set defaults -target="$(cat "$__object/parameter/target" 2>/dev/null \ - || echo "/target" | tee "$__object/parameter/target")" +# TODO is this neccesary or should this be handled using the usual parameter/default workflow? +if [ ! -f "$__object/parameter/target" ]; then + echo "/target" > "$__object/parameter/target" +fi diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index d8bc5eb7..8ce77768 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -87,6 +87,7 @@ if [ $(expr "${ip}" : ".*, .*") -gt "0" ]; then SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do + # TODO BUG? Why is cur_ip unused in the following line? # Just get the last IP address for SSH to listen on mgmt_ip=$(echo "${ip}" | cut '-d ' -f1) # In case using "ip netmask" format rather than CIDR done diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index d883dec4..c76bb014 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -92,6 +92,7 @@ if [ $(expr "${ip}" : ".*|.*") -gt "0" ]; then SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do + # TODO BUG? Why is cur_ip unused in the following line? # Just get the last IP address for SSH to listen on mgmt_ip=$(echo "${ip}" | sed -E -e 's/^.*\|(.*)\/[0-9]+$/\1/') done diff --git a/cdist/conf/type/__pacman_conf_integrate/manifest b/cdist/conf/type/__pacman_conf_integrate/manifest index b26bca50..829fb34b 100755 --- a/cdist/conf/type/__pacman_conf_integrate/manifest +++ b/cdist/conf/type/__pacman_conf_integrate/manifest @@ -20,8 +20,6 @@ state=$(cat $__object/parameter/state 2>/dev/null) -path="/etc/" - if [ "${state}" = "present" ]; then __file /etc/pacman.conf\ --owner root --group root --mode 644 --source $__type/files/pacman.conf.cdist diff --git a/cdist/conf/type/__postfix_master/manifest b/cdist/conf/type/__postfix_master/manifest index 4991a13d..94b5044f 100755 --- a/cdist/conf/type/__postfix_master/manifest +++ b/cdist/conf/type/__postfix_master/manifest @@ -36,7 +36,6 @@ __postfix # Default to object_id service="$(cat "$__object/parameter/service" 2>/dev/null || echo "$__object_id")" -state="$(cat "$__object/parameter/state")" # NOTE: keep variables in sync in manifest,explorer,gencode-* prefix="#cdist:$__object_name" diff --git a/cdist/conf/type/__qemu_img/manifest b/cdist/conf/type/__qemu_img/manifest index e7417389..55f3bf16 100755 --- a/cdist/conf/type/__qemu_img/manifest +++ b/cdist/conf/type/__qemu_img/manifest @@ -4,7 +4,6 @@ # Default settings # -format="$(cat "$__object/parameter/format")" state_should="$(cat "$__object/parameter/state")" diskimage="/$__object_id" diff --git a/cdist/conf/type/__rvm_gem/gencode-remote b/cdist/conf/type/__rvm_gem/gencode-remote index 1fe6e78e..9212de91 100755 --- a/cdist/conf/type/__rvm_gem/gencode-remote +++ b/cdist/conf/type/__rvm_gem/gencode-remote @@ -20,8 +20,6 @@ gem="$__object_id" gemset="$(cat "$__object/parameter/gemset")" -ruby="$(echo "$gemset" | cut -d '@' -f 1)" -gemsetname="$(echo "$gemset" | cut -d '@' -f 2)" state_is="$(cat "$__object/explorer/state")" user="$(cat "$__object/parameter/user")" state_should="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__rvm_gemset/explorer/state b/cdist/conf/type/__rvm_gemset/explorer/state index fa643a6e..d1462134 100755 --- a/cdist/conf/type/__rvm_gemset/explorer/state +++ b/cdist/conf/type/__rvm_gemset/explorer/state @@ -18,9 +18,6 @@ # along with cdist. If not, see . # -gemset="$__object_id" -ruby="$(echo "$gemset" | cut -d '@' -f 1)" -gemsetname="$(echo "$gemset" | cut -d '@' -f2)" user="$(cat "$__object/parameter/user")" if [ ! -e "~$user/.rvm/scripts/rvm" ] ; then diff --git a/cdist/conf/type/__rvm_ruby/gencode-remote b/cdist/conf/type/__rvm_ruby/gencode-remote index 9bbc6031..f2fd41ef 100755 --- a/cdist/conf/type/__rvm_ruby/gencode-remote +++ b/cdist/conf/type/__rvm_ruby/gencode-remote @@ -21,7 +21,6 @@ ruby="$__object_id" state_is="$(cat "$__object/explorer/state")" user="$(cat "$__object/parameter/user")" -default="$(cat "$__object/parameter/default" 2>/dev/null || true)" state_should="$(cat "$__object/parameter/state")" [ "$state_is" = "$state_should" ] && exit 0 diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local index 18bf09f5..851970e8 100755 --- a/cdist/conf/type/__staged_file/gencode-local +++ b/cdist/conf/type/__staged_file/gencode-local @@ -23,7 +23,6 @@ destination="$__object_id" source="$(cat "$__object/parameter/source")" -cksum="$(cat "$__object/parameter/cksum")" stage_dir="$(cat "$__object/parameter/stage-dir")" state="$(cat "$__object/parameter/state")" fetch_command="$(cat "$__object/parameter/fetch-command")" diff --git a/cdist/conf/type/__staged_file/manifest b/cdist/conf/type/__staged_file/manifest index 1654e1d9..c8e1fbbb 100755 --- a/cdist/conf/type/__staged_file/manifest +++ b/cdist/conf/type/__staged_file/manifest @@ -19,11 +19,7 @@ # destination="$__object_id" -source="$(cat "$__object/parameter/source")" -cksum="$(cat "$__object/parameter/cksum")" stage_dir="$(cat "$__object/parameter/stage-dir")" -state="$(cat "$__object/parameter/state")" -fetch_command="$(cat "$__object/parameter/fetch-command")" stage_file="${stage_dir}/${destination}" set -- "/${destination}" From b3f4d3849a687a1795302cb25d3c50c251a4417c Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 13:34:11 +0200 Subject: [PATCH 124/178] Fix __install_umount target parameter --- cdist/conf/type/__install_umount/manifest | 25 ------------------- .../__install_umount/parameter/default/target | 1 + .../type/__install_umount/parameter/optional | 1 + 3 files changed, 2 insertions(+), 25 deletions(-) delete mode 100755 cdist/conf/type/__install_umount/manifest create mode 100644 cdist/conf/type/__install_umount/parameter/default/target create mode 100644 cdist/conf/type/__install_umount/parameter/optional diff --git a/cdist/conf/type/__install_umount/manifest b/cdist/conf/type/__install_umount/manifest deleted file mode 100755 index d900df1b..00000000 --- a/cdist/conf/type/__install_umount/manifest +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -e -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -# set defaults -# TODO is this neccesary or should this be handled using the usual parameter/default workflow? -if [ ! -f "$__object/parameter/target" ]; then - echo "/target" > "$__object/parameter/target" -fi diff --git a/cdist/conf/type/__install_umount/parameter/default/target b/cdist/conf/type/__install_umount/parameter/default/target new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/cdist/conf/type/__install_umount/parameter/default/target @@ -0,0 +1 @@ +/target diff --git a/cdist/conf/type/__install_umount/parameter/optional b/cdist/conf/type/__install_umount/parameter/optional new file mode 100644 index 00000000..eb5a316c --- /dev/null +++ b/cdist/conf/type/__install_umount/parameter/optional @@ -0,0 +1 @@ +target From 352679386dc9acd18b7e98de0dfb3d2b8a75b81c Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 13:36:45 +0200 Subject: [PATCH 125/178] Drop unused options parameter in __install_reboot --- cdist/conf/type/__install_reboot/man.rst | 3 +-- cdist/conf/type/__install_reboot/manifest | 25 ----------------------- 2 files changed, 1 insertion(+), 27 deletions(-) delete mode 100755 cdist/conf/type/__install_reboot/manifest diff --git a/cdist/conf/type/__install_reboot/man.rst b/cdist/conf/type/__install_reboot/man.rst index ecf78ca7..9a53b37a 100644 --- a/cdist/conf/type/__install_reboot/man.rst +++ b/cdist/conf/type/__install_reboot/man.rst @@ -18,8 +18,7 @@ None OPTIONAL PARAMETERS ------------------- -options - options to pass to the reboot command. e.g. -f +None EXAMPLES diff --git a/cdist/conf/type/__install_reboot/manifest b/cdist/conf/type/__install_reboot/manifest deleted file mode 100755 index 46a6356e..00000000 --- a/cdist/conf/type/__install_reboot/manifest +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -e -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -# set defaults -# TODO is this neccesary or should this be handled using the usual parameter/default workflow? -if [ ! -f "$__object/parameter/options" ]; then - touch "$__object/parameter/options" -fi From 23debd5b6f27bcafd6b91149af9280f4398016df Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 19:21:59 +0200 Subject: [PATCH 126/178] Use cur_ip in jail for freebsd --- cdist/conf/type/__jail_freebsd10/gencode-remote | 3 +-- cdist/conf/type/__jail_freebsd9/gencode-remote | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 8ce77768..5be04bc6 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -87,9 +87,8 @@ if [ $(expr "${ip}" : ".*, .*") -gt "0" ]; then SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do - # TODO BUG? Why is cur_ip unused in the following line? # Just get the last IP address for SSH to listen on - mgmt_ip=$(echo "${ip}" | cut '-d ' -f1) # In case using "ip netmask" format rather than CIDR + mgmt_ip=$(echo "${cur_ip}" | cut '-d ' -f1) # In case using "ip netmask" format rather than CIDR done IFS="$SAVE_IFS" else diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index c76bb014..177bdf74 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -92,9 +92,8 @@ if [ $(expr "${ip}" : ".*|.*") -gt "0" ]; then SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do - # TODO BUG? Why is cur_ip unused in the following line? # Just get the last IP address for SSH to listen on - mgmt_ip=$(echo "${ip}" | sed -E -e 's/^.*\|(.*)\/[0-9]+$/\1/') + mgmt_ip=$(echo "${cur_ip}" | sed -E -e 's/^.*\|(.*)\/[0-9]+$/\1/') done IFS="$SAVE_IFS" else From ae7ccc59959ce45948bd6b767d8559ff9715cdbf Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Tue, 2 Oct 2018 21:44:27 +0200 Subject: [PATCH 127/178] Fix SC2086 on all scripts Relevant documentation: https://github.com/koalaman/shellcheck/wiki/SC2086 relates to: #540 This commit addresses 241 separate cases of missing quotes around variables. --- cdist/conf/explorer/cpu_cores | 2 +- cdist/conf/explorer/cpu_sockets | 2 +- cdist/conf/explorer/lsb_codename | 2 +- cdist/conf/explorer/lsb_description | 2 +- cdist/conf/explorer/lsb_id | 2 +- cdist/conf/explorer/lsb_release | 2 +- cdist/conf/explorer/os_version | 2 +- cdist/conf/type/__apt_ppa/gencode-remote | 4 ++-- .../type/__ccollect_source/gencode-remote | 16 ++++++------- cdist/conf/type/__ccollect_source/manifest | 2 +- cdist/conf/type/__cron/explorer/entry | 4 ++-- cdist/conf/type/__daemontools/manifest | 4 ++-- .../conf/type/__daemontools_service/manifest | 4 ++-- cdist/conf/type/__directory/gencode-remote | 16 ++++++------- cdist/conf/type/__docker/manifest | 10 ++++---- .../conf/type/__docker_compose/gencode-remote | 4 ++-- cdist/conf/type/__docker_compose/manifest | 4 ++-- cdist/conf/type/__file/gencode-remote | 16 ++++++------- .../conf/type/__firewalld_rule/explorer/rule | 2 +- .../conf/type/__firewalld_rule/gencode-remote | 8 +++---- cdist/conf/type/__git/explorer/group | 2 +- cdist/conf/type/__git/explorer/owner | 2 +- cdist/conf/type/__grafana_dashboard/manifest | 4 ++-- cdist/conf/type/__group/explorer/gshadow | 2 +- cdist/conf/type/__group/gencode-remote | 8 +++---- .../__install_bootloader_grub/gencode-remote | 2 +- .../__install_generate_fstab/gencode-local | 4 ++-- .../gencode-remote | 14 +++++------ cdist/conf/type/__jail/manifest | 2 +- cdist/conf/type/__link/explorer/type | 2 +- .../conf/type/__package/explorer/pkgng_exists | 2 +- .../type/__package_luarocks/gencode-remote | 4 ++-- cdist/conf/type/__package_opkg/gencode-remote | 4 ++-- .../conf/type/__package_pacman/gencode-remote | 4 ++-- .../type/__package_pkg_freebsd/gencode-remote | 2 +- .../type/__package_rubygem/gencode-remote | 4 ++-- .../__package_update_index/explorer/currage | 2 +- .../type/__package_update_index/explorer/type | 2 +- .../__package_update_index/gencode-remote | 2 +- .../type/__package_upgrade_all/gencode-remote | 4 ++-- cdist/conf/type/__package_yum/gencode-remote | 4 ++-- .../conf/type/__package_zypper/gencode-remote | 6 ++--- cdist/conf/type/__pacman_conf/manifest | 24 +++++++++---------- .../type/__pacman_conf_integrate/manifest | 10 ++++---- cdist/conf/type/__pf_apply/explorer/rcvar | 2 +- cdist/conf/type/__pf_ruleset/explorer/cksum | 2 +- cdist/conf/type/__pf_ruleset/explorer/rcvar | 2 +- .../type/__prometheus_alertmanager/manifest | 2 +- .../conf/type/__prometheus_exporter/manifest | 8 +++---- cdist/conf/type/__prometheus_server/manifest | 6 ++--- cdist/conf/type/__pyvenv/explorer/group | 2 +- cdist/conf/type/__pyvenv/explorer/owner | 2 +- cdist/conf/type/__pyvenv/gencode-remote | 2 +- cdist/conf/type/__qemu_img/gencode-remote | 2 +- cdist/conf/type/__rsync/gencode-local | 2 +- cdist/conf/type/__rvm/explorer/state | 2 +- .../type/__ssh_authorized_key/explorer/entry | 2 +- .../type/__ssh_authorized_key/gencode-remote | 2 +- .../conf/type/__start_on_boot/gencode-remote | 10 ++++---- cdist/conf/type/__start_on_boot/manifest | 2 +- cdist/conf/type/__user/explorer/shadow | 2 +- cdist/conf/type/__user/gencode-remote | 6 ++--- cdist/conf/type/__zypper_repo/gencode-remote | 8 +++---- .../__zypper_service/explorer/service_uri | 2 +- .../conf/type/__zypper_service/gencode-remote | 16 ++++++------- 65 files changed, 154 insertions(+), 154 deletions(-) diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores index 2f676f7a..27cc6800 100755 --- a/cdist/conf/explorer/cpu_cores +++ b/cdist/conf/explorer/cpu_cores @@ -31,7 +31,7 @@ case "$os" in *) if [ -r /proc/cpuinfo ]; then cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)" - if [ ${cores} -eq 0 ]; then + if [ "${cores}" -eq 0 ]; then cores="1" fi echo "$cores" diff --git a/cdist/conf/explorer/cpu_sockets b/cdist/conf/explorer/cpu_sockets index 2d577043..a32e2f00 100755 --- a/cdist/conf/explorer/cpu_sockets +++ b/cdist/conf/explorer/cpu_sockets @@ -31,7 +31,7 @@ case "$os" in *) if [ -r /proc/cpuinfo ]; then sockets="$(grep "physical id" /proc/cpuinfo | sort -u | wc -l)" - if [ ${sockets} -eq 0 ]; then + if [ "${sockets}" -eq 0 ]; then sockets="$(grep -c "processor" /proc/cpuinfo)" fi echo "${sockets}" diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index eebd3e0f..bfabd444 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -20,7 +20,7 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") ;; diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index 23f45421..441fdbf3 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -20,7 +20,7 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") ;; diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 9754eb63..1a12b610 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -20,7 +20,7 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) (. /etc/openwrt_release && echo "$DISTRIB_ID") ;; diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 35b5547c..25eb95b1 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -20,7 +20,7 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") ;; diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 380782cc..4c41695b 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -22,7 +22,7 @@ # # -case "$($__explorer/os)" in +case "$("$__explorer/os")" in amazon) cat /etc/system-release ;; diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index f60cb7ac..25dec58d 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,9 +29,9 @@ fi case "$state_should" in present) - echo add-apt-repository \"$name\" + echo "add-apt-repository \"$name\"" ;; absent) - echo remove-apt-repository \"$name\" + echo "remove-apt-repository \"$name\"" ;; esac diff --git a/cdist/conf/type/__ccollect_source/gencode-remote b/cdist/conf/type/__ccollect_source/gencode-remote index 763f219e..5f2f97e3 100755 --- a/cdist/conf/type/__ccollect_source/gencode-remote +++ b/cdist/conf/type/__ccollect_source/gencode-remote @@ -42,18 +42,18 @@ get_current_value() { } set_group() { - echo chgrp \"$1\" \"$destination\" - echo chgrp $1 >> "$__messages_out" + echo chgrp \""$1"\" \""$destination"\" + echo chgrp "$1" >> "$__messages_out" } set_owner() { - echo chown \"$1\" \"$destination\" - echo chown $1 >> "$__messages_out" + echo chown \""$1"\" \""$destination"\" + echo chown "$1" >> "$__messages_out" } set_mode() { - echo chmod \"$1\" \"$destination\" - echo chmod $1 >> "$__messages_out" + echo chmod \""$1"\" \""$destination"\" + echo chmod "$1" >> "$__messages_out" } set_attributes= @@ -67,7 +67,7 @@ case "$state_should" in # change 0xxx format to xxx format => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" fi value_is="$(get_current_value "$attribute" "$value_should")" @@ -81,7 +81,7 @@ case "$state_should" in absent) if [ "$type" = "file" ]; then - echo rm -f \"$destination\" + echo rm -f \""$destination"\" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__ccollect_source/manifest b/cdist/conf/type/__ccollect_source/manifest index 26c6cc99..727a4c97 100755 --- a/cdist/conf/type/__ccollect_source/manifest +++ b/cdist/conf/type/__ccollect_source/manifest @@ -55,5 +55,5 @@ if [ -f "$__object/parameter/exclude" ]; then fi if [ -f "$__object/parameter/create-destination" ]; then - __directory "${destination}" --parents --state ${state} + __directory "${destination}" --parents --state "${state}" fi diff --git a/cdist/conf/type/__cron/explorer/entry b/cdist/conf/type/__cron/explorer/entry index 2167e045..801861a3 100644 --- a/cdist/conf/type/__cron/explorer/entry +++ b/cdist/conf/type/__cron/explorer/entry @@ -24,7 +24,7 @@ user="$(cat "$__object/parameter/user")" if [ -f "$__object/parameter/raw_command" ]; then command="$(cat "$__object/parameter/command")" - crontab -u $user -l 2>/dev/null | grep "^$command\$" || true + crontab -u "$user" -l 2>/dev/null | grep "^$command\$" || true else - crontab -u $user -l 2>/dev/null | grep "# $name\$" || true + crontab -u "$user" -l 2>/dev/null | grep "# $name\$" || true fi diff --git a/cdist/conf/type/__daemontools/manifest b/cdist/conf/type/__daemontools/manifest index 45ce3df6..656f4984 100755 --- a/cdist/conf/type/__daemontools/manifest +++ b/cdist/conf/type/__daemontools/manifest @@ -3,8 +3,8 @@ pkg=$(cat "$__object/parameter/from-package") servicedir=$(cat "$__object/parameter/servicedir") -__package $pkg -__directory $servicedir --mode 700 +__package "$pkg" +__directory "$servicedir" --mode 700 os=$(cat "$__global/explorer/os") init=$(cat "$__global/explorer/init") diff --git a/cdist/conf/type/__daemontools_service/manifest b/cdist/conf/type/__daemontools_service/manifest index 9e8e0bee..78bae285 100755 --- a/cdist/conf/type/__daemontools_service/manifest +++ b/cdist/conf/type/__daemontools_service/manifest @@ -25,14 +25,14 @@ badusage() { [ -z "$run$runfile" ] && badusage [ -n "$run" ] && [ -n "$runfile" ] && badusage -__directory $servicedir/$name/log/main --parents +__directory "$servicedir/$name/log/main" --parents echo "$RUN_PREFIX$run" | require="__directory/$servicedir/$name/log/main" __config_file "$servicedir/$name/run" \ --onchange "svc -t '$servicedir/$name' 2>/dev/null" \ --mode 755 \ --source "${runfile:--}" -echo "$RUN_PREFIX$logrun" | require="__directory/$servicedir/$name/log/main" __config_file $servicedir/$name/log/run \ +echo "$RUN_PREFIX$logrun" | require="__directory/$servicedir/$name/log/main" __config_file "$servicedir/$name/log/run" \ --onchange "svc -t '$servicedir/$name/log' 2>/dev/null" \ --mode 755 \ --source "-" diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index cced4624..35866eb4 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -57,18 +57,18 @@ get_current_value() { } set_group() { - echo chgrp $recursive \"$1\" \"$destination\" - echo chgrp $recursive $1 >> "$__messages_out" + echo chgrp "$recursive" \""$1"\" \""$destination"\" + echo chgrp "$recursive" "$1" >> "$__messages_out" } set_owner() { - echo chown $recursive \"$1\" \"$destination\" - echo chown $recursive $1 >> "$__messages_out" + echo chown "$recursive" \""$1"\" \""$destination"\" + echo chown "$recursive" "$1" >> "$__messages_out" } set_mode() { - echo chmod $recursive \"$1\" \"$destination\" - echo chmod $recursive $1 >> "$__messages_out" + echo chmod "$recursive" \""$1"\" \""$destination"\" + echo chmod "$recursive" "$1" >> "$__messages_out" } case "$state_should" in @@ -94,7 +94,7 @@ case "$state_should" in # change 0xxx format to xxx format => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" fi if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then @@ -105,7 +105,7 @@ case "$state_should" in ;; absent) if [ "$type" = "directory" ]; then - echo rm -rf \"$destination\" + echo rm -rf \""$destination"\" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index e0558b46..35760cef 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -31,8 +31,8 @@ case "$os" in --enabled \ --gpgcheck 1 \ --gpgkey 'https://download.docker.com/linux/centos/gpg' \ - --state ${state} - require="__yum_repo/docker-ce-stable" __package docker-ce --state ${state} + --state "${state}" + require="__yum_repo/docker-ce-stable" __package docker-ce --state "${state}" else echo "CentOS version 7 is required!" >&2 exit 1 @@ -45,14 +45,14 @@ case "$os" in __package gnupg2 fi __apt_key_uri docker --name "Docker Release (CE deb) " \ - --uri "https://download.docker.com/linux/${os}/gpg" --state ${state} + --uri "https://download.docker.com/linux/${os}/gpg" --state "${state}" export CDIST_ORDER_DEPENDENCY=on __apt_source docker \ --uri "https://download.docker.com/linux/${os}" \ --distribution "$(cat "$__global/explorer/lsb_codename")" \ - --state ${state} \ + --state "${state}" \ --component "stable" - __package docker-ce --state ${state} + __package docker-ce --state "${state}" unset CDIST_ORDER_DEPENDENCY ;; *) diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index 2b8267a9..396c93ac 100755 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -22,9 +22,9 @@ version="$(cat "$__object/parameter/version")" state="$(cat "$__object/parameter/state")" -if [ ${state} = "present" ]; then +if [ "${state}" = "present" ]; then # Download docker-compose file - echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' + echo 'curl -L "https://github.com/docker/compose/releases/download/'"${version}"'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' # Change permissions echo 'chmod +x /usr/local/bin/docker-compose' diff --git a/cdist/conf/type/__docker_compose/manifest b/cdist/conf/type/__docker_compose/manifest index c17f0f33..f7de3a76 100755 --- a/cdist/conf/type/__docker_compose/manifest +++ b/cdist/conf/type/__docker_compose/manifest @@ -22,10 +22,10 @@ state="$(cat "$__object/parameter/state")" # Needed packages -if [ ${state} = "present" ]; then +if [ "${state}" = "present" ]; then __docker __package curl -elif [ ${state} = "absent" ]; then +elif [ "${state}" = "absent" ]; then __file /usr/local/bin/docker-compose --state absent else echo "Unknown state: ${state}" >&2 diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 9dfd1833..ebc30fd9 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -43,18 +43,18 @@ get_current_value() { } set_group() { - echo chgrp \"$1\" \"$destination\" - echo chgrp $1 >> "$__messages_out" + echo chgrp \""$1"\" \""$destination"\" + echo chgrp "$1" >> "$__messages_out" } set_owner() { - echo chown \"$1\" \"$destination\" - echo chown $1 >> "$__messages_out" + echo chown \""$1"\" \""$destination"\" + echo chown "$1" >> "$__messages_out" } set_mode() { - echo chmod \"$1\" \"$destination\" - echo chmod $1 >> "$__messages_out" + echo chmod \""$1"\" \""$destination"\" + echo chmod "$1" >> "$__messages_out" } set_attributes= @@ -68,7 +68,7 @@ case "$state_should" in # change 0xxx format to xxx format => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" fi value_is="$(get_current_value "$attribute" "$value_should")" @@ -82,7 +82,7 @@ case "$state_should" in absent) if [ "$type" = "file" ]; then - echo rm -f \"$destination\" + echo rm -f \""$destination"\" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__firewalld_rule/explorer/rule b/cdist/conf/type/__firewalld_rule/explorer/rule index 5a0e0265..0234e5b6 100644 --- a/cdist/conf/type/__firewalld_rule/explorer/rule +++ b/cdist/conf/type/__firewalld_rule/explorer/rule @@ -25,7 +25,7 @@ chain="$(cat "$__object/parameter/chain")" priority="$(cat "$__object/parameter/priority")" rule="$(cat "$__object/parameter/rule")" -if firewall-cmd --permanent --direct --query-rule "$protocol" "$table" "$chain" "$priority" $rule >/dev/null; then +if firewall-cmd --permanent --direct --query-rule "$protocol" "$table" "$chain" "$priority" "$rule" >/dev/null; then echo present else echo absent diff --git a/cdist/conf/type/__firewalld_rule/gencode-remote b/cdist/conf/type/__firewalld_rule/gencode-remote index 4c824d39..bc218d9a 100755 --- a/cdist/conf/type/__firewalld_rule/gencode-remote +++ b/cdist/conf/type/__firewalld_rule/gencode-remote @@ -33,13 +33,13 @@ rule="$(cat "$__object/parameter/rule")" case "$state_should" in present) - echo firewall-cmd --quiet --permanent --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule - echo firewall-cmd --quiet --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule + echo "firewall-cmd --quiet --permanent --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" + echo "firewall-cmd --quiet --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" ;; absent) - echo firewall-cmd --quiet --permanent --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule - echo firewall-cmd --quiet --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule + echo "firewall-cmd --quiet --permanent --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" + echo "firewall-cmd --quiet --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" ;; *) echo "Unknown state $state_should" >&2 diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 1308c710..3ddf9656 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -2,4 +2,4 @@ destination="/$__object_id/.git" -stat --print "%G" ${destination} 2>/dev/null || exit 0 +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 8c36b035..4c3cd431 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -2,4 +2,4 @@ destination="/$__object_id/.git" -stat --print "%U" ${destination} 2>/dev/null || exit 0 +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 308af59a..2e9bd115 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -1,7 +1,7 @@ #!/bin/sh -e -os=$(cat $__global/explorer/os) -os_version=$(cat $__global/explorer/os_version) +os=$(cat "$__global/explorer/os") +os_version=$(cat "$__global/explorer/os_version") case $os in debian|devuan) diff --git a/cdist/conf/type/__group/explorer/gshadow b/cdist/conf/type/__group/explorer/gshadow index 2e2ab29d..ef40b7bc 100755 --- a/cdist/conf/type/__group/explorer/gshadow +++ b/cdist/conf/type/__group/explorer/gshadow @@ -22,7 +22,7 @@ # name=$__object_id -os="$($__explorer/os)" +os="$("$__explorer/os")" case "$os" in "freebsd"|"netbsd") diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote index 68475178..5b3947dc 100755 --- a/cdist/conf/type/__group/gencode-remote +++ b/cdist/conf/type/__group/gencode-remote @@ -63,8 +63,8 @@ if [ "$state" = "present" ]; then ;; esac if [ "$new_value" != "$current_value" ]; then - set -- "$@" "$(shorten_property $property)" \'$new_value\' - echo change $property $new_value $current_value >> "$__messages_out" + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' + echo "change $property $new_value $current_value" >> "$__messages_out" fi fi done @@ -83,9 +83,9 @@ if [ "$state" = "present" ]; then new_value="$(cat "$__object/parameter/$property")" if [ -z "$new_value" ]; then # Boolean parameters have no value - set -- "$@" "$(shorten_property $property)" + set -- "$@" "$(shorten_property "$property")" else - set -- "$@" "$(shorten_property $property)" \'$new_value\' + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' fi fi done diff --git a/cdist/conf/type/__install_bootloader_grub/gencode-remote b/cdist/conf/type/__install_bootloader_grub/gencode-remote index 6e6e5e85..1caebbbf 100755 --- a/cdist/conf/type/__install_bootloader_grub/gencode-remote +++ b/cdist/conf/type/__install_bootloader_grub/gencode-remote @@ -28,7 +28,7 @@ install_script="$__object/files/install_script" # Link file descriptor #6 with stdout exec 6>&1 # Link stdout with $install_script -exec > $install_script +exec > "$install_script" # Generate script to install bootloader on distro printf '#!/bin/sh -l\n' diff --git a/cdist/conf/type/__install_generate_fstab/gencode-local b/cdist/conf/type/__install_generate_fstab/gencode-local index 5cc7d877..d0f99f61 100755 --- a/cdist/conf/type/__install_generate_fstab/gencode-local +++ b/cdist/conf/type/__install_generate_fstab/gencode-local @@ -23,7 +23,7 @@ cat "$__type/files/fstab.header" > "$destination" mkdir "$__object/files" # get current UUID's from target_host -$__remote_exec $__target_host blkid > "$__object/files/blkid" +$__remote_exec "$__target_host" blkid > "$__object/files/blkid" for object in $(find "$__global/object/__install_mount" -type d -name "$__cdist_object_marker"); do device="$(cat "$object/parameter/device")" @@ -54,7 +54,7 @@ for object in $(find "$__global/object/__install_mount" -type d -name "$__cdist_ ;; esac if [ -f "$__object/parameter/uuid" ]; then - uuid="$(grep -w $device "$__object/files/blkid" | awk '{print $2}')" + uuid="$(grep -w "$device" "$__object/files/blkid" | awk '{print $2}')" if [ -n "$uuid" ]; then echo "# $dir was on $device during installation" >> "$destination" device="$uuid" diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index 9e5fcd00..b8100c52 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -34,9 +34,9 @@ size_to_mb() { size=$1 available_size="$2" - number_suffix="$(echo ${size} | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" - number="$(echo ${number_suffix} | cut -d '|' -f1)" - suffix="$(echo ${number_suffix} | cut -d '|' -f2)" + number_suffix="$(echo "${size}" | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" + number="$(echo "${number_suffix}" | cut -d '|' -f1)" + suffix="$(echo "${number_suffix}" | cut -d '|' -f2)" case "$suffix" in K|k) @@ -62,10 +62,10 @@ get_objects() { for object in $(find "$__global/object/__install_partition_msdos" -type d -name "$__cdist_object_marker"); do object_device="$(cat "$object/parameter/device")" object_minor="$(cat "$object/parameter/minor")" - echo "$object_device $object_minor $object" >> $objects_file + echo "$object_device $object_minor $object" >> "$objects_file" done - sort -k 1,2 $objects_file | cut -d' ' -f 3 - rm $objects_file + sort -k 1,2 "$objects_file" | cut -d' ' -f 3 + rm "$objects_file" unset objects_file unset object unset object_device @@ -87,7 +87,7 @@ for object in $objects; do if [ "$current_device" != "$device" ]; then echo "create_disklabel \"$device\" || die 'Failed to create disklabel for $device'" current_device="$device" - device_name=$(echo ${device} | sed -e 's:^/dev/::;s:/:\\/:g') + device_name=$(echo "${device}" | sed -e 's:^/dev/::;s:/:\\/:g') available_device_size=$(( $(awk "/${device_name}\$/ { print \$3; }" "$partitions") / 1024)) # make sure we don't go past the end of the drive available_device_size=$((available_device_size - 2)) diff --git a/cdist/conf/type/__jail/manifest b/cdist/conf/type/__jail/manifest index 7564be26..222cb859 100755 --- a/cdist/conf/type/__jail/manifest +++ b/cdist/conf/type/__jail/manifest @@ -35,7 +35,7 @@ fi jaildir="$(cat "$__object/parameter/jaildir")" -__directory ${jaildir} --parents +__directory "${jaildir}" --parents set -- "$@" "$__object_id" "--state" "$state" cd "$__object/parameter" diff --git a/cdist/conf/type/__link/explorer/type b/cdist/conf/type/__link/explorer/type index 579fd081..48a79bf0 100755 --- a/cdist/conf/type/__link/explorer/type +++ b/cdist/conf/type/__link/explorer/type @@ -32,7 +32,7 @@ elif [ -f "$destination" ]; then case "$type" in hard) link_count=$(ls -l "$destination" | awk '{ print $2 }') - if [ $link_count -gt 1 ]; then + if [ "$link_count" -gt 1 ]; then echo hardlink exit 0 fi diff --git a/cdist/conf/type/__package/explorer/pkgng_exists b/cdist/conf/type/__package/explorer/pkgng_exists index 355c5d65..6d69ba14 100755 --- a/cdist/conf/type/__package/explorer/pkgng_exists +++ b/cdist/conf/type/__package/explorer/pkgng_exists @@ -21,7 +21,7 @@ # Retrieve the status of a package - parsed dpkg output # -if [ "$($__explorer/os)" = "freebsd" ]; then +if [ "$("$__explorer/os")" = "freebsd" ]; then command -v pkg fi diff --git a/cdist/conf/type/__package_luarocks/gencode-remote b/cdist/conf/type/__package_luarocks/gencode-remote index cae06b22..e14d7a8e 100755 --- a/cdist/conf/type/__package_luarocks/gencode-remote +++ b/cdist/conf/type/__package_luarocks/gencode-remote @@ -42,10 +42,10 @@ fi case "$state_should" in present) - echo luarocks install \"$name\" + echo luarocks install \""$name"\" ;; absent) - echo luarocks remove \"$name\" + echo luarocks remove \""$name"\" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index 09fe69a4..53f154a5 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -45,10 +45,10 @@ case "$state_should" in if [ "$present" = "notpresent" ]; then echo opkg --verbosity=0 update fi - echo opkg --verbosity=0 install \"$name\" + echo opkg --verbosity=0 install \""$name"\" ;; absent) - echo opkg --verbosity=0 remove \"$name\" + echo opkg --verbosity=0 remove \""$name"\" ;; *) echo "Unknown state: $state" >&2 diff --git a/cdist/conf/type/__package_pacman/gencode-remote b/cdist/conf/type/__package_pacman/gencode-remote index 69a5d62a..43649124 100755 --- a/cdist/conf/type/__package_pacman/gencode-remote +++ b/cdist/conf/type/__package_pacman/gencode-remote @@ -45,10 +45,10 @@ fi case "$state_should" in present) - echo pacman --needed --noconfirm --noprogressbar -S \"$name\" + echo pacman --needed --noconfirm --noprogressbar -S \""$name"\" ;; absent) - echo pacman --noconfirm --noprogressbar -R \"$name\" + echo pacman --noconfirm --noprogressbar -R \""$name"\" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 012bf2ad..6c6e15b1 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -33,7 +33,7 @@ assert () # If condition false, lineno=$2 - if [ ! $1 ] + if [ ! "$1" ] then echo "Assertion failed: \"$1\"" echo "File \"$0\", line $lineno, called by $(caller 0)" diff --git a/cdist/conf/type/__package_rubygem/gencode-remote b/cdist/conf/type/__package_rubygem/gencode-remote index 6d793ac0..d474db20 100755 --- a/cdist/conf/type/__package_rubygem/gencode-remote +++ b/cdist/conf/type/__package_rubygem/gencode-remote @@ -39,10 +39,10 @@ fi case "$state_should" in present) - echo gem install \"$name\" --no-ri --no-rdoc + echo gem install \""$name"\" --no-ri --no-rdoc ;; absent) - echo gem uninstall \"$name\" + echo gem uninstall \""$name"\" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage index 50474fb3..3539b8e1 100644 --- a/cdist/conf/type/__package_update_index/explorer/currage +++ b/cdist/conf/type/__package_update_index/explorer/currage @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -type="$($__type_explorer/type)" +type="$("$__type_explorer/type")" case "$type" in apt) diff --git a/cdist/conf/type/__package_update_index/explorer/type b/cdist/conf/type/__package_update_index/explorer/type index 805b9f04..35254c5f 100644 --- a/cdist/conf/type/__package_update_index/explorer/type +++ b/cdist/conf/type/__package_update_index/explorer/type @@ -21,7 +21,7 @@ if [ -f "$__object/parameter/type" ]; then cat "$__object/parameter/type" else # By default determine package manager based on operating system - os="$($__explorer/os)" + os="$("$__explorer/os")" case "$os" in amazon|scientific|centos|fedora|redhat) echo "yum" ;; debian|ubuntu|devuan) echo "apt" ;; diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 8589c6d1..738d38eb 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -31,7 +31,7 @@ if [ -n "$maxage" ]; then if [ "$type" != "apt" ] && [ "$type" != "pacman" ]; then echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2 exit 1 - elif [ $currage -lt $maxage ]; then + elif [ "$currage" -lt "$maxage" ]; then exit 0 # no need to update fi fi diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index bcad8a43..38aa001e 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -53,8 +53,8 @@ case "$type" in ;; apt) if [ -f "$apt_dist_upgrade" ] - then echo $aptget dist-upgrade - else echo $aptget upgrade + then echo "$aptget dist-upgrade" + else echo "$aptget upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_yum/gencode-remote b/cdist/conf/type/__package_yum/gencode-remote index e1323dea..d860c48c 100755 --- a/cdist/conf/type/__package_yum/gencode-remote +++ b/cdist/conf/type/__package_yum/gencode-remote @@ -60,10 +60,10 @@ fi case "$state_should" in present) - echo yum $opts install \"$install_name\" + echo "yum $opts install \"$install_name\"" ;; absent) - echo yum $opts remove \"$name\" + echo "yum $opts remove \"$name\"" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_zypper/gencode-remote b/cdist/conf/type/__package_zypper/gencode-remote index d9372b6d..5b1b763e 100755 --- a/cdist/conf/type/__package_zypper/gencode-remote +++ b/cdist/conf/type/__package_zypper/gencode-remote @@ -61,15 +61,15 @@ case "$state_should" in present) if [ -z "$version_should" ]; then [ "$state_is" = "present" ] && exit 0 # if state is present, we dont need to do anything - echo zypper $globalopts install --type \"$ptype\" --auto-agree-with-licenses \"$name\" ">/dev/null" + echo "zypper $globalopts install --type \"$ptype\" --auto-agree-with-licenses \"$name\" >/dev/null" else [ "$state_is" = "present" ] && [ "$version_should" = "$version_is" ] && exit 0 # if state is present and version is correct, we dont need to do anything - echo zypper $globalopts install --oldpackage --type \"$ptype\" --auto-agree-with-licenses \"$name\" = \"$version_should\" ">/dev/null" + echo "zypper $globalopts install --oldpackage --type \"$ptype\" --auto-agree-with-licenses \"$name\" = \"$version_should\" >/dev/null" fi ;; absent) [ "$state_is" = "absent" ] && exit 0 # if state is absent, we dont need to do anything - echo zypper $globalopts remove --type \"$ptype\" \"$name\" ">/dev/null" + echo "zypper $globalopts remove --type \"$ptype\" \"$name\" >/dev/null" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__pacman_conf/manifest b/cdist/conf/type/__pacman_conf/manifest index 1561d613..a43f18a1 100755 --- a/cdist/conf/type/__pacman_conf/manifest +++ b/cdist/conf/type/__pacman_conf/manifest @@ -59,13 +59,13 @@ if [ "${file}" ]; then if [ "${state}" = "present" ]; then - require="__file/${sec_path}/plain_file_${file}" __key_value ${file}_${key}\ - --file ${sec_path}/plain_file_${file} --key ${key} --value ${value} --delimiter ' = ' + require="__file/${sec_path}/plain_file_${file}" __key_value "${file}_${key}" \ + --file "${sec_path}/plain_file_${file}" --key "${key}" --value "${value}" --delimiter ' = ' exit 0 elif [ "${state}" = "absent" ]; then - require="__file/${sec_path}/plain_file_${file}" __key_value ${file}_${key}\ + require="__file/${sec_path}/plain_file_${file}" __key_value "${file}_${key}" \ --state absent exit 0 @@ -87,19 +87,19 @@ eof if [ "${MATCH}" -eq 1 ]; then if [ "${value}" = "on" ]; then - require="__file/${sec_path}/${section}" __line ${key}_${value}\ - --file ${sec_path}/${section} --line ${key} + require="__file/${sec_path}/${section}" __line "${key}_${value}" \ + --file "${sec_path}/${section}" --line "${key}" elif [ "${value}" = "off" ]; then - require="__file/${sec_path}/${section}" __line ${key}_${value}\ - --file ${sec_path}/${section} --line ${key} --state absent + require="__file/${sec_path}/${section}" __line "${key}_${value}" \ + --file "${sec_path}/${section}" --line "${key}" --state absent fi else contains_element "${key}" "${allowed_option_keys}" if [ "${MATCH}" -eq 1 ]; then - require="__file/${sec_path}/${section}" __key_value ${section}_${key}\ - --file ${sec_path}/${section} --key ${key} --value ${value} --delimiter ' = ' + require="__file/${sec_path}/${section}" __key_value "${section}_${key}" \ + --file "${sec_path}/${section}" --key "${key}" --value "${value}" --delimiter ' = ' else echo "Key: ${key} is not valid. Have a look at man pacman.conf" >&2 fi @@ -118,12 +118,12 @@ eof exit fi - require="__file/${sec_path}/repo_${section}" __key_value ${section}_${key}\ - --file ${sec_path}/repo_${section} --key ${key} --value ${value} --delimiter ' = ' + require="__file/${sec_path}/repo_${section}" __key_value "${section}_${key}" \ + --file "${sec_path}/repo_${section}" --key "${key}" --value "${value}" --delimiter ' = ' elif [ "${state}" = "absent" ]; then - require="__file/${sec_path}/repo_${section}" __key_value ${section}_${key}\ + require="__file/${sec_path}/repo_${section}" __key_value "${section}_${key}" \ --state absent else diff --git a/cdist/conf/type/__pacman_conf_integrate/manifest b/cdist/conf/type/__pacman_conf_integrate/manifest index b26bca50..884af74d 100755 --- a/cdist/conf/type/__pacman_conf_integrate/manifest +++ b/cdist/conf/type/__pacman_conf_integrate/manifest @@ -18,16 +18,16 @@ # along with cdist. If not, see . # -state=$(cat $__object/parameter/state 2>/dev/null) +state=$(cat "$__object/parameter/state" 2>/dev/null) path="/etc/" if [ "${state}" = "present" ]; then __file /etc/pacman.conf\ - --owner root --group root --mode 644 --source $__type/files/pacman.conf.cdist + --owner root --group root --mode 644 --source "$__type/files/pacman.conf.cdist" __file /etc/pacman.d/options\ - --owner root --group root --mode 644 --source $__type/files/options + --owner root --group root --mode 644 --source "$__type/files/options" __file /etc/pacman.d/repo_empty_placeholder\ --owner root --group root --mode 644 @@ -38,10 +38,10 @@ if [ "${state}" = "present" ]; then elif [ "${state}" = "absent" ]; then __file /etc/pacman.conf\ - --owner root --group root --mode 644 --source $__type/files/pacman.conf.pacman + --owner root --group root --mode 644 --source "$__type/files/pacman.conf.pacman" __file /etc/pacman.d/mirrorlist\ - --owner root --group root --mode 644 --source $__type/files/mirrorlist + --owner root --group root --mode 644 --source "$__type/files/mirrorlist" __file /etc/pacman.d/options\ --state absent diff --git a/cdist/conf/type/__pf_apply/explorer/rcvar b/cdist/conf/type/__pf_apply/explorer/rcvar index 20e9dfcc..7c8d535f 100755 --- a/cdist/conf/type/__pf_apply/explorer/rcvar +++ b/cdist/conf/type/__pf_apply/explorer/rcvar @@ -29,7 +29,7 @@ RC="/etc/rc.conf" PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" -echo ${PFCONF:-"/etc/pf.conf"} +echo "${PFCONF:-"/etc/pf.conf"}" # Debug #set +x diff --git a/cdist/conf/type/__pf_ruleset/explorer/cksum b/cdist/conf/type/__pf_ruleset/explorer/cksum index f8679836..9be6c901 100755 --- a/cdist/conf/type/__pf_ruleset/explorer/cksum +++ b/cdist/conf/type/__pf_ruleset/explorer/cksum @@ -33,7 +33,7 @@ TMP="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" PFCONF="${TMP:-"/etc/pf.conf"}" if [ -f "${PFCONF}" ]; then # The pf config file exists, find its cksum. - cksum -o 1 ${PFCONF} | cut -d= -f2 | awk '{print $1}' + cksum -o 1 "${PFCONF}" | cut -d= -f2 | awk '{print $1}' fi # Debug diff --git a/cdist/conf/type/__pf_ruleset/explorer/rcvar b/cdist/conf/type/__pf_ruleset/explorer/rcvar index 20e9dfcc..7c8d535f 100755 --- a/cdist/conf/type/__pf_ruleset/explorer/rcvar +++ b/cdist/conf/type/__pf_ruleset/explorer/rcvar @@ -29,7 +29,7 @@ RC="/etc/rc.conf" PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" -echo ${PFCONF:-"/etc/pf.conf"} +echo "${PFCONF:-"/etc/pf.conf"}" # Debug #set +x diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index 054e44cb..3f417526 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -57,7 +57,7 @@ __key_value alertmanager_args --file /etc/default/prometheus-alertmanager \ require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ - --source $config \ + --source "$config" \ --group prometheus --mode 640 \ --onchange "service prometheus-alertmanager reload" # TODO when a config-check tool is available, check config here diff --git a/cdist/conf/type/__prometheus_exporter/manifest b/cdist/conf/type/__prometheus_exporter/manifest index ae4ed94a..35e654e2 100644 --- a/cdist/conf/type/__prometheus_exporter/manifest +++ b/cdist/conf/type/__prometheus_exporter/manifest @@ -2,7 +2,7 @@ export GOBIN=/opt/gocode/bin # where to find go binaries -exporter="$(cat $__object/parameter/exporter)" +exporter="$(cat "$__object/parameter/exporter")" [ -z "$exporter" ] && exporter="$__object_id" __user prometheus --system @@ -18,7 +18,7 @@ case $exporter in ;; blackbox) require="$require __daemontools_service/${exporter}-exporter __user/prometheus" __config_file "/service/${exporter}-exporter/blackbox.yml" \ - --source $__type/files/blackbox.yml \ + --source "$__type/files/blackbox.yml" \ --group prometheus --mode 640 \ --onchange "svc -h /service/${exporter}-exporter" require="$require __golang_from_vendor" __go_get github.com/prometheus/blackbox_exporter @@ -39,9 +39,9 @@ case $exporter in ;; esac -require="$require __daemontools" __daemontools_service ${exporter}-exporter --run "$run" +require="$require __daemontools" __daemontools_service "${exporter}-exporter" --run "$run" if [ -f "$__object/parameter/add-consul-service" ]; then - __consul_service ${exporter}-exporter --port $port --check-http "http://localhost:$port/metrics" --check-interval 10s + __consul_service "${exporter}-exporter" --port "$port" --check-http "http://localhost:$port/metrics" --check-interval 10s fi #__daemontools --install-init-script diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index e2f32fd5..63b7b438 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -13,7 +13,7 @@ storage_path="$(cat "$__object/parameter/storage-path")" rule_files="$(cat "$__object/parameter/rule-files")" # explorer in kB => convert; by default we go with 1/2 RAM -[ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat $__global/explorer/memory)*1024/2)) +[ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat "$__global/explorer/memory")*1024/2)) ##### INSTALL THE PACKAGE ################################################### @@ -55,12 +55,12 @@ __key_value prometheus_args --file /etc/default/prometheus \ require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ - --source $config \ + --source "$config" \ --group prometheus --mode 640 \ --onchange "promtool check config $CONF && service prometheus reload" for file in $rule_files; do - dest=$CONF_DIR/$(basename $file) + dest=$CONF_DIR/$(basename "$file") require="$require $require_pkg" \ __config_file "$dest" \ --source "$file" \ diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index ff072c5e..a655bda7 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -2,4 +2,4 @@ destination="/$__object_id" -stat --print "%G" ${destination} 2>/dev/null || exit 0 +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index b77e3c6e..8b3c7f8e 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -2,4 +2,4 @@ destination="/$__object_id" -stat --print "%U" ${destination} 2>/dev/null || exit 0 +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote index 4cba5987..04700683 100755 --- a/cdist/conf/type/__pyvenv/gencode-remote +++ b/cdist/conf/type/__pyvenv/gencode-remote @@ -47,7 +47,7 @@ fi case $state_should in present) if [ "$state_should" != "$state_is" ]; then - echo $pyvenv $venvparams "$destination" + echo "$pyvenv $venvparams $destination" fi if { [ -n "$owner" ] && [ "$owner_is" != "$owner" ]; } || \ { [ -n "$group" ] && [ "$group_is" != "$group" ]; }; then diff --git a/cdist/conf/type/__qemu_img/gencode-remote b/cdist/conf/type/__qemu_img/gencode-remote index 9127e5ef..95d07b7a 100755 --- a/cdist/conf/type/__qemu_img/gencode-remote +++ b/cdist/conf/type/__qemu_img/gencode-remote @@ -18,4 +18,4 @@ format="$(cat "$__object/parameter/format")" size="$(cat "$__object/parameter/size")" diskimage="/$__object_id" -echo qemu-img create -f \"$format\" \"$diskimage\" \"$size\" +echo "qemu-img create -f \"$format\" \"$diskimage\" \"$size\"" diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 155f3a3a..c7196175 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -31,7 +31,7 @@ set -- if [ -f "$__object/parameter/rsync-opts" ]; then while read opts; do set -- "$@" "--$opts" - done < $__object/parameter/rsync-opts + done < "$__object/parameter/rsync-opts" fi echo rsync -a \ diff --git a/cdist/conf/type/__rvm/explorer/state b/cdist/conf/type/__rvm/explorer/state index f43f5509..74d17048 100755 --- a/cdist/conf/type/__rvm/explorer/state +++ b/cdist/conf/type/__rvm/explorer/state @@ -28,7 +28,7 @@ if [ "$user" = "root" ]; then echo absent fi else - if su - $user -c "[ -d \"\$HOME/.rvm\" ]" ; then + if su - "$user" -c "[ -d \"\$HOME/.rvm\" ]" ; then echo "present" else echo "absent" diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index 157d70f1..b528b26e 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -24,7 +24,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ] # in greping everything in file and all entries from file are removed. if [ -n "${type_and_key}" ] then - file="$(cat $__object/parameter/file)" + file="$(cat "$__object/parameter/file")" # get any entries that match the type and key grep ".*$type_and_key\([ \n]\|$\)" "$file" || true diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 333dfa03..325854c2 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -88,7 +88,7 @@ fi entry="$(cat "$__object/files/should")" state_should="$(cat "$__object/parameter/state")" num_existing_entries=$(grep -c -F -x "$entry" "$__object/explorer/entry" || true) -if [ $num_existing_entries -eq 1 ]; then +if [ "$num_existing_entries" -eq 1 ]; then state_is="present" else # Posix grep does not define the -m option, so we can not remove a single diff --git a/cdist/conf/type/__start_on_boot/gencode-remote b/cdist/conf/type/__start_on_boot/gencode-remote index 14ee7dab..0ecea85b 100755 --- a/cdist/conf/type/__start_on_boot/gencode-remote +++ b/cdist/conf/type/__start_on_boot/gencode-remote @@ -59,11 +59,11 @@ case "$state_should" in ;; gentoo) - echo rc-update add \"$name\" \"$target_runlevel\" + echo "rc-update add \"$name\" \"$target_runlevel\"" ;; amazon|scientific|centos|fedora|owl|redhat|suse) - echo chkconfig \"$name\" on + echo "chkconfig \"$name\" on" ;; openwrt) @@ -98,15 +98,15 @@ case "$state_should" in else case "$os" in debian|ubuntu|devuan) - echo update-rc.d -f \"$name\" remove + echo "update-rc.d -f \"$name\" remove" ;; gentoo) - echo rc-update del \"$name\" \"$target_runlevel\" + echo "rc-update del \"$name\" \"$target_runlevel\"" ;; centos|fedora|owl|redhat|suse) - echo chkconfig \"$name\" off + echo "chkconfig \"$name\" off" ;; openwrt) diff --git a/cdist/conf/type/__start_on_boot/manifest b/cdist/conf/type/__start_on_boot/manifest index b9ee20e2..c1c983ec 100644 --- a/cdist/conf/type/__start_on_boot/manifest +++ b/cdist/conf/type/__start_on_boot/manifest @@ -16,7 +16,7 @@ case "$os" in else value='NO' fi - __key_value rcconf-$name-enable \ + __key_value "rcconf-$name-enable" \ --file /etc/rc.conf \ --key "${name}_enable" \ --value "\"$value\"" \ diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow index 1e6658d4..c49992d5 100755 --- a/cdist/conf/type/__user/explorer/shadow +++ b/cdist/conf/type/__user/explorer/shadow @@ -22,7 +22,7 @@ # name=$__object_id -os="$($__explorer/os)" +os="$("$__explorer/os")" # Default to using shadow passwords database="shadow" diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index 332e93f1..23f01114 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -97,7 +97,7 @@ if [ "$state" = "present" ]; then fi if [ "$new_value" != "$current_value" ]; then - set -- "$@" "$(shorten_property $property)" \'$new_value\' + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' fi done @@ -118,9 +118,9 @@ if [ "$state" = "present" ]; then [ "$property" = "remove-home" ] && continue new_value="$(cat "$property")" if [ -z "$new_value" ];then # Boolean values have no value - set -- "$@" "$(shorten_property $property)" + set -- "$@" "$(shorten_property "$property")" else - set -- "$@" "$(shorten_property $property)" \'$new_value\' + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' fi done diff --git a/cdist/conf/type/__zypper_repo/gencode-remote b/cdist/conf/type/__zypper_repo/gencode-remote index 94c3f146..b8eeef0f 100755 --- a/cdist/conf/type/__zypper_repo/gencode-remote +++ b/cdist/conf/type/__zypper_repo/gencode-remote @@ -70,25 +70,25 @@ case "$state" in fi if [ -z "$repo_id" ]; then # Repo not present, so we need to create it - echo zypper $zypper_def_opts addrepo "'$uri'" "'$desc'" + echo "zypper $zypper_def_opts addrepo '$uri' '$desc'" fi ;; absent) if [ ! -z "$act_id" ]; then # Repo present (act_id not ""), so we ned to delete it - echo zypper $zypper_def_opts removerepo "$act_id" + echo "zypper $zypper_def_opts removerepo $act_id" fi ;; enabled) if [ ! -z "$act_id" ] && [ "$repostate" = "disabled" ]; then # Repo present (act_id not "") and repostate not enabled, so a enable call is needed - echo zypper $zypper_def_opts modifyrepo -e "$act_id" + echo "zypper $zypper_def_opts modifyrepo -e $act_id" fi ;; disabled) if [ ! -z "$act_id" ] && [ "$repostate" = "enabled" ]; then # Repo present (act_id not "") and repostate enabled, so a disable call is needed - echo zypper $zypper_def_opts modifyrepo -d "$act_id" + echo "zypper $zypper_def_opts modifyrepo -d $act_id" fi ;; *) diff --git a/cdist/conf/type/__zypper_service/explorer/service_uri b/cdist/conf/type/__zypper_service/explorer/service_uri index b8de0dcd..2476ab71 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_uri +++ b/cdist/conf/type/__zypper_service/explorer/service_uri @@ -25,4 +25,4 @@ if [ -f "$__object/parameter/uri" ]; then else uri="/$__object_id" fi -zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'$uri'" {print $NF}' +zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'"$uri"'" {print $NF}' diff --git a/cdist/conf/type/__zypper_service/gencode-remote b/cdist/conf/type/__zypper_service/gencode-remote index 955698d1..6701347f 100755 --- a/cdist/conf/type/__zypper_service/gencode-remote +++ b/cdist/conf/type/__zypper_service/gencode-remote @@ -59,10 +59,10 @@ if [ -f "$__object/parameter/remove-all-other-services" ]; then # file exists -> True for i in $exp_ids; do if [ "$i" != "$exp_id" ] ; then - echo zypper $zypper_def_opts removeservice "$i" "&>/dev/null" + echo "zypper $zypper_def_opts removeservice $i &>/dev/null" fi done - echo zypper $zypper_def_opts refs "&>/dev/null" + echo "zypper $zypper_def_opts refs &>/dev/null" fi @@ -71,14 +71,14 @@ fi case "$state_should" in present) - echo zypper $zypper_def_opts addservice -t "$stype" "$uri" \"$desc\" - echo zypper $zypper_def_opts refs - echo zypper $zypper_def_opts ref + echo "zypper $zypper_def_opts addservice -t $stype $uri \"$desc\"" + echo "zypper $zypper_def_opts refs" + echo "zypper $zypper_def_opts ref" ;; absent) - echo zypper $zypper_def_opts removeservice "$service_id" - echo zypper $zypper_def_opts refs - echo zypper $zypper_def_opts ref + echo "zypper $zypper_def_opts removeservice $service_id" + echo "zypper $zypper_def_opts refs" + echo "zypper $zypper_def_opts ref" ;; *) echo "Unknown state: $state_should" >&2 From 53728a7ff0c1ac16d7dddb827915390d2c59abe9 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Wed, 3 Oct 2018 19:38:47 +0200 Subject: [PATCH 128/178] Change backslash-dquote to single quote Slightly breaking change, but improves security and readability. --- cdist/conf/type/__apt_ppa/gencode-remote | 4 +-- .../type/__ccollect_source/gencode-remote | 14 +++++----- cdist/conf/type/__directory/gencode-remote | 18 ++++++------- cdist/conf/type/__file/gencode-remote | 14 +++++----- .../conf/type/__firewalld_rule/gencode-remote | 8 +++--- .../gencode-remote | 2 +- cdist/conf/type/__package_apt/gencode-remote | 4 +-- .../conf/type/__package_emerge/gencode-remote | 4 +-- .../type/__package_luarocks/gencode-remote | 4 +-- cdist/conf/type/__package_opkg/gencode-remote | 4 +-- .../conf/type/__package_pacman/gencode-remote | 4 +-- cdist/conf/type/__package_pip/gencode-remote | 4 +-- .../type/__package_rubygem/gencode-remote | 4 +-- cdist/conf/type/__package_yum/gencode-remote | 4 +-- .../conf/type/__package_zypper/gencode-remote | 6 ++--- cdist/conf/type/__qemu_img/gencode-remote | 2 +- .../conf/type/__start_on_boot/gencode-remote | 26 +++++++++---------- .../conf/type/__zypper_service/gencode-remote | 2 +- 18 files changed, 64 insertions(+), 64 deletions(-) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 25dec58d..84ebebfe 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,9 +29,9 @@ fi case "$state_should" in present) - echo "add-apt-repository \"$name\"" + echo "add-apt-repository '$name'" ;; absent) - echo "remove-apt-repository \"$name\"" + echo "remove-apt-repository '$name'" ;; esac diff --git a/cdist/conf/type/__ccollect_source/gencode-remote b/cdist/conf/type/__ccollect_source/gencode-remote index 5f2f97e3..5daa3fd3 100755 --- a/cdist/conf/type/__ccollect_source/gencode-remote +++ b/cdist/conf/type/__ccollect_source/gencode-remote @@ -42,18 +42,18 @@ get_current_value() { } set_group() { - echo chgrp \""$1"\" \""$destination"\" - echo chgrp "$1" >> "$__messages_out" + echo "chgrp '$1' '$destination'" + echo "chgrp '$1'" >> "$__messages_out" } set_owner() { - echo chown \""$1"\" \""$destination"\" - echo chown "$1" >> "$__messages_out" + echo "chown '$1' '$destination'" + echo "chown '$1'" >> "$__messages_out" } set_mode() { - echo chmod \""$1"\" \""$destination"\" - echo chmod "$1" >> "$__messages_out" + echo "chmod '$1' '$destination'" + echo "chmod '$1'" >> "$__messages_out" } set_attributes= @@ -81,7 +81,7 @@ case "$state_should" in absent) if [ "$type" = "file" ]; then - echo rm -f \""$destination"\" + echo "rm -f '$destination'" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 35866eb4..4220c6fb 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -57,18 +57,18 @@ get_current_value() { } set_group() { - echo chgrp "$recursive" \""$1"\" \""$destination"\" - echo chgrp "$recursive" "$1" >> "$__messages_out" + echo "chgrp '$recursive' '$1' '$destination'" + echo "chgrp '$recursive' '$1'" >> "$__messages_out" } set_owner() { - echo chown "$recursive" \""$1"\" \""$destination"\" - echo chown "$recursive" "$1" >> "$__messages_out" + echo "chown '$recursive' '$1' '$destination'" + echo "chown '$recursive' '$1'" >> "$__messages_out" } set_mode() { - echo chmod "$recursive" \""$1"\" \""$destination"\" - echo chmod "$recursive" "$1" >> "$__messages_out" + echo "chmod '$recursive' '$1' '$destination'" + echo "chmod '$recursive' '$1'" >> "$__messages_out" } case "$state_should" in @@ -78,10 +78,10 @@ case "$state_should" in if [ "$type" != "none" ]; then # our destination is not a directory, remove whatever is there # and then create our directory and set all attributes - echo rm -f "\"$destination\"" + echo "rm -f '$destination'" echo "remove non directory" >> "$__messages_out" fi - echo "mkdir $mkdiropt \"$destination\"" + echo "mkdir $mkdiropt '$destination'" echo "create" >> "$__messages_out" fi @@ -105,7 +105,7 @@ case "$state_should" in ;; absent) if [ "$type" = "directory" ]; then - echo rm -rf \""$destination"\" + echo "rm -rf '$destination'" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index ebc30fd9..ab9c2fd1 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -43,18 +43,18 @@ get_current_value() { } set_group() { - echo chgrp \""$1"\" \""$destination"\" - echo chgrp "$1" >> "$__messages_out" + echo "chgrp '$1' '$destination'" + echo "chgrp '$1'" >> "$__messages_out" } set_owner() { - echo chown \""$1"\" \""$destination"\" - echo chown "$1" >> "$__messages_out" + echo "chown '$1' '$destination'" + echo "chown '$1'" >> "$__messages_out" } set_mode() { - echo chmod \""$1"\" \""$destination"\" - echo chmod "$1" >> "$__messages_out" + echo "chmod '$1' '$destination'" + echo "chmod '$1'" >> "$__messages_out" } set_attributes= @@ -82,7 +82,7 @@ case "$state_should" in absent) if [ "$type" = "file" ]; then - echo rm -f \""$destination"\" + echo "rm -f '$destination'" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__firewalld_rule/gencode-remote b/cdist/conf/type/__firewalld_rule/gencode-remote index bc218d9a..225a0502 100755 --- a/cdist/conf/type/__firewalld_rule/gencode-remote +++ b/cdist/conf/type/__firewalld_rule/gencode-remote @@ -33,13 +33,13 @@ rule="$(cat "$__object/parameter/rule")" case "$state_should" in present) - echo "firewall-cmd --quiet --permanent --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" - echo "firewall-cmd --quiet --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" + echo "firewall-cmd --quiet --permanent --direct --add-rule '$protocol' '$table' '$chain' '$priority' $rule" + echo "firewall-cmd --quiet --direct --add-rule '$protocol' '$table' '$chain' '$priority' $rule" ;; absent) - echo "firewall-cmd --quiet --permanent --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" - echo "firewall-cmd --quiet --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule" + echo "firewall-cmd --quiet --permanent --direct --remove-rule '$protocol' '$table' '$chain' '$priority' $rule" + echo "firewall-cmd --quiet --direct --remove-rule '$protocol' '$table' '$chain' '$priority' $rule" ;; *) echo "Unknown state $state_should" >&2 diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index b8100c52..644de33c 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -85,7 +85,7 @@ primary_count=0 for object in $objects; do device="$(cat "$object/parameter/device")" if [ "$current_device" != "$device" ]; then - echo "create_disklabel \"$device\" || die 'Failed to create disklabel for $device'" + echo "create_disklabel '$device' || die 'Failed to create disklabel for $device'" current_device="$device" device_name=$(echo "${device}" | sed -e 's:^/dev/::;s:/:\\/:g') available_device_size=$(( $(awk "/${device_name}\$/ { print \$3; }" "$partitions") / 1024)) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index cbe14787..699eb0c9 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -77,11 +77,11 @@ case "$state_should" in if [ -n "$version" ]; then name="${name}=${version}" fi - echo "$aptget install $target_release \"$name\"" + echo "$aptget install $target_release '$name'" echo "installed" >> "$__messages_out" ;; absent) - echo "$aptget remove $purgeparam \"$name\"" + echo "$aptget remove $purgeparam '$name'" echo "removed" >> "$__messages_out" ;; *) diff --git a/cdist/conf/type/__package_emerge/gencode-remote b/cdist/conf/type/__package_emerge/gencode-remote index ada2b414..8abcb5b1 100755 --- a/cdist/conf/type/__package_emerge/gencode-remote +++ b/cdist/conf/type/__package_emerge/gencode-remote @@ -63,10 +63,10 @@ fi case "$state_should" in present) - echo "emerge \"$name\" &>/dev/null || exit 1" + echo "emerge '$name' &>/dev/null || exit 1" ;; absent) - echo "emerge -C \"$name\" &>/dev/null || exit 1" + echo "emerge -C '$name' &>/dev/null || exit 1" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_luarocks/gencode-remote b/cdist/conf/type/__package_luarocks/gencode-remote index e14d7a8e..60b9bfea 100755 --- a/cdist/conf/type/__package_luarocks/gencode-remote +++ b/cdist/conf/type/__package_luarocks/gencode-remote @@ -42,10 +42,10 @@ fi case "$state_should" in present) - echo luarocks install \""$name"\" + echo "luarocks install '$name'" ;; absent) - echo luarocks remove \""$name"\" + echo "luarocks remove '$name'" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index 53f154a5..7013ec9f 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -45,10 +45,10 @@ case "$state_should" in if [ "$present" = "notpresent" ]; then echo opkg --verbosity=0 update fi - echo opkg --verbosity=0 install \""$name"\" + echo "opkg --verbosity=0 install '$name'" ;; absent) - echo opkg --verbosity=0 remove \""$name"\" + echo "opkg --verbosity=0 remove '$name'" ;; *) echo "Unknown state: $state" >&2 diff --git a/cdist/conf/type/__package_pacman/gencode-remote b/cdist/conf/type/__package_pacman/gencode-remote index 43649124..7ba85479 100755 --- a/cdist/conf/type/__package_pacman/gencode-remote +++ b/cdist/conf/type/__package_pacman/gencode-remote @@ -45,10 +45,10 @@ fi case "$state_should" in present) - echo pacman --needed --noconfirm --noprogressbar -S \""$name"\" + echo "pacman --needed --noconfirm --noprogressbar -S '$name'" ;; absent) - echo pacman --noconfirm --noprogressbar -R \""$name"\" + echo "pacman --noconfirm --noprogressbar -R '$name'" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 933406f2..33556bec 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -53,7 +53,7 @@ case "$state_should" in present) if [ "$runas" ] then - echo "su -c \"$pip install -q $name\" $runas" + echo "su -c '$pip install -q $name' $runas" else echo $pip install -q "$name" fi @@ -61,7 +61,7 @@ case "$state_should" in absent) if [ "$runas" ] then - echo "su -c \"$pip uninstall -q -y $name\" $runas" + echo "su -c '$pip uninstall -q -y $name' $runas" else echo $pip uninstall -q -y "$name" fi diff --git a/cdist/conf/type/__package_rubygem/gencode-remote b/cdist/conf/type/__package_rubygem/gencode-remote index d474db20..ee563ef8 100755 --- a/cdist/conf/type/__package_rubygem/gencode-remote +++ b/cdist/conf/type/__package_rubygem/gencode-remote @@ -39,10 +39,10 @@ fi case "$state_should" in present) - echo gem install \""$name"\" --no-ri --no-rdoc + echo "gem install '$name' --no-ri --no-rdoc" ;; absent) - echo gem uninstall \""$name"\" + echo "gem uninstall '$name'" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_yum/gencode-remote b/cdist/conf/type/__package_yum/gencode-remote index d860c48c..97265827 100755 --- a/cdist/conf/type/__package_yum/gencode-remote +++ b/cdist/conf/type/__package_yum/gencode-remote @@ -60,10 +60,10 @@ fi case "$state_should" in present) - echo "yum $opts install \"$install_name\"" + echo "yum $opts install '$install_name'" ;; absent) - echo "yum $opts remove \"$name\"" + echo "yum $opts remove '$name'" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_zypper/gencode-remote b/cdist/conf/type/__package_zypper/gencode-remote index 5b1b763e..1f15c531 100755 --- a/cdist/conf/type/__package_zypper/gencode-remote +++ b/cdist/conf/type/__package_zypper/gencode-remote @@ -61,15 +61,15 @@ case "$state_should" in present) if [ -z "$version_should" ]; then [ "$state_is" = "present" ] && exit 0 # if state is present, we dont need to do anything - echo "zypper $globalopts install --type \"$ptype\" --auto-agree-with-licenses \"$name\" >/dev/null" + echo "zypper $globalopts install --type '$ptype' --auto-agree-with-licenses '$name' >/dev/null" else [ "$state_is" = "present" ] && [ "$version_should" = "$version_is" ] && exit 0 # if state is present and version is correct, we dont need to do anything - echo "zypper $globalopts install --oldpackage --type \"$ptype\" --auto-agree-with-licenses \"$name\" = \"$version_should\" >/dev/null" + echo "zypper $globalopts install --oldpackage --type '$ptype' --auto-agree-with-licenses '$name' = '$version_should' >/dev/null" fi ;; absent) [ "$state_is" = "absent" ] && exit 0 # if state is absent, we dont need to do anything - echo "zypper $globalopts remove --type \"$ptype\" \"$name\" >/dev/null" + echo "zypper $globalopts remove --type '$ptype' '$name' >/dev/null" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__qemu_img/gencode-remote b/cdist/conf/type/__qemu_img/gencode-remote index 95d07b7a..94816f58 100755 --- a/cdist/conf/type/__qemu_img/gencode-remote +++ b/cdist/conf/type/__qemu_img/gencode-remote @@ -18,4 +18,4 @@ format="$(cat "$__object/parameter/format")" size="$(cat "$__object/parameter/size")" diskimage="/$__object_id" -echo "qemu-img create -f \"$format\" \"$diskimage\" \"$size\"" +echo "qemu-img create -f '$format' '$diskimage' '$size'" diff --git a/cdist/conf/type/__start_on_boot/gencode-remote b/cdist/conf/type/__start_on_boot/gencode-remote index 0ecea85b..122692ec 100755 --- a/cdist/conf/type/__start_on_boot/gencode-remote +++ b/cdist/conf/type/__start_on_boot/gencode-remote @@ -37,16 +37,16 @@ case "$state_should" in if [ "$init" = 'systemd' ]; then # this handles ALL linux distros with systemd # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions - echo "systemctl -q enable \"$name\"" + echo "systemctl -q enable '$name'" else case "$os" in debian) case "$os_version" in [1-7]*) - echo "update-rc.d \"$name\" defaults >/dev/null" + echo "update-rc.d '$name' defaults >/dev/null" ;; 8*) - echo "systemctl enable \"$name\"" + echo "systemctl enable '$name'" ;; *) echo "Unsupported version $os_version of $os" >&2 @@ -55,26 +55,26 @@ case "$state_should" in esac ;; devuan) - echo "update-rc.d \"$name\" defaults >/dev/null" + echo "update-rc.d '$name' defaults >/dev/null" ;; gentoo) - echo "rc-update add \"$name\" \"$target_runlevel\"" + echo "rc-update add '$name' '$target_runlevel'" ;; amazon|scientific|centos|fedora|owl|redhat|suse) - echo "chkconfig \"$name\" on" + echo "chkconfig '$name' on" ;; openwrt) # 'enable' can be successful and still return a non-zero exit # code, deal with it by checking for success ourselves in that # case (the || ... part). - echo "/etc/init.d/\"$name\" enable || [ -f /etc/rc.d/S??\"$name\" ]" + echo "'/etc/init.d/$name' enable || [ -f /etc/rc.d/S??'$name' ]" ;; ubuntu) - echo "update-rc.d \"$name\" defaults >/dev/null" + echo "update-rc.d '$name' defaults >/dev/null" ;; freebsd) @@ -93,24 +93,24 @@ case "$state_should" in if [ "$init" = 'systemd' ]; then # this handles ALL linux distros with systemd # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions - echo "systemctl -q disable \"$name\"" + echo "systemctl -q disable '$name'" else case "$os" in debian|ubuntu|devuan) - echo "update-rc.d -f \"$name\" remove" + echo "update-rc.d -f '$name' remove" ;; gentoo) - echo "rc-update del \"$name\" \"$target_runlevel\"" + echo "rc-update del '$name' '$target_runlevel'" ;; centos|fedora|owl|redhat|suse) - echo "chkconfig \"$name\" off" + echo "chkconfig '$name' off" ;; openwrt) - echo "\"/etc/init.d/$name\" disable" + echo "'/etc/init.d/$name' disable" ;; *) diff --git a/cdist/conf/type/__zypper_service/gencode-remote b/cdist/conf/type/__zypper_service/gencode-remote index 6701347f..59ffc942 100755 --- a/cdist/conf/type/__zypper_service/gencode-remote +++ b/cdist/conf/type/__zypper_service/gencode-remote @@ -71,7 +71,7 @@ fi case "$state_should" in present) - echo "zypper $zypper_def_opts addservice -t $stype $uri \"$desc\"" + echo "zypper $zypper_def_opts addservice -t $stype $uri '$desc'" echo "zypper $zypper_def_opts refs" echo "zypper $zypper_def_opts ref" ;; From fff656d1acc596646249eb31903b5bcbb6003caf Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 19:28:36 +0200 Subject: [PATCH 129/178] Unquote $recursive to support empty-case This is safe to do, as the value of $recursive is not dependent on user-supplied code. --- cdist/conf/type/__directory/gencode-remote | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 4220c6fb..374db47a 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -57,18 +57,18 @@ get_current_value() { } set_group() { - echo "chgrp '$recursive' '$1' '$destination'" - echo "chgrp '$recursive' '$1'" >> "$__messages_out" + echo "chgrp $recursive '$1' '$destination'" + echo "chgrp $recursive '$1'" >> "$__messages_out" } set_owner() { - echo "chown '$recursive' '$1' '$destination'" - echo "chown '$recursive' '$1'" >> "$__messages_out" + echo "chown $recursive '$1' '$destination'" + echo "chown $recursive '$1'" >> "$__messages_out" } set_mode() { - echo "chmod '$recursive' '$1' '$destination'" - echo "chmod '$recursive' '$1'" >> "$__messages_out" + echo "chmod $recursive '$1' '$destination'" + echo "chmod $recursive '$1'" >> "$__messages_out" } case "$state_should" in From e2593c0dcbababa2d0f0cb1fd0d29dbbb5da24bb Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 21:23:12 +0200 Subject: [PATCH 130/178] Fix SC2215 (missing brackets on if) This might have never worked as intended, but how would you know? --- cdist/conf/type/__rvm_gemset/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rvm_gemset/gencode-remote b/cdist/conf/type/__rvm_gemset/gencode-remote index 78851f9a..3cdc66a6 100755 --- a/cdist/conf/type/__rvm_gemset/gencode-remote +++ b/cdist/conf/type/__rvm_gemset/gencode-remote @@ -33,7 +33,7 @@ case "$state_should" in cat << DONE su - "$user" -c "source ~/.rvm/scripts/rvm; rvm $gemset --create" DONE - if -f "$__object/parameter/default"; then + if [ -f "$__object/parameter/default" ]; then cat << DONE su - "$user" -c "source ~/.rvm/scripts/rvm; rvm use --default $gemset" DONE From a6fdf4255aa877850890330c91d9330617b99fb3 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 21:17:31 +0200 Subject: [PATCH 131/178] Fix SC2069 (wrong stdout/stderr redirect order) In the original order, stderr was connected to the old stdout (terminal). This was _probably_ not intended. The new order fixes this by first connecting stdout to /dev/null and then attaching stderr to that as well. --- cdist/conf/explorer/machine | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine b/cdist/conf/explorer/machine index d4a0e106..7ecb67e3 100755 --- a/cdist/conf/explorer/machine +++ b/cdist/conf/explorer/machine @@ -22,6 +22,6 @@ # # -if command -v uname 2>&1 >/dev/null; then +if command -v uname >/dev/null 2>&1 ; then uname -m fi From a0db375f91bad2e288a36ce298146e22672718d9 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 21:30:28 +0200 Subject: [PATCH 132/178] Disable SC2059 (printf injection) for staged_file We're explicitly allowing for 'printf injection' here, so it's ok to just disable the corresponding shellcheck check. --- cdist/conf/type/__staged_file/gencode-local | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local index 18bf09f5..c0d94e92 100755 --- a/cdist/conf/type/__staged_file/gencode-local +++ b/cdist/conf/type/__staged_file/gencode-local @@ -57,6 +57,7 @@ get_file() { } fetch_file() { + # shellcheck disable=SC2059 printf "$fetch_command" "$source" printf ' > "%s"\n' "$stage_file" } @@ -64,8 +65,10 @@ fetch_file() { fetch_and_prepare_file() { printf 'tmpdir="$(mktemp -d --tmpdir="/tmp" "%s")"\n' "${__type##*/}.XXXXXXXXXX" printf 'cd "$tmpdir"\n' + # shellcheck disable=SC2059 printf "$fetch_command > \"%s\"\n" "$source" "$source_file_name" prepare_command="$(cat "$__object/parameter/prepare-command")" + # shellcheck disable=SC2059 printf "$prepare_command > \"%s\"\n" "$source_file_name" "$stage_file" printf 'cd - >/dev/null\n' printf 'rm -rf "$tmpdir"\n' From bcd7fbbcc48b72eb0f5ec378f0555428a0dac8d8 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 22:01:25 +0200 Subject: [PATCH 133/178] Fix SC2155 (directly export computed variables) --- cdist/conf/type/__key_value/explorer/state | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__key_value/explorer/state b/cdist/conf/type/__key_value/explorer/state index b990733d..7b2de1df 100755 --- a/cdist/conf/type/__key_value/explorer/state +++ b/cdist/conf/type/__key_value/explorer/state @@ -19,9 +19,9 @@ # along with cdist. If not, see . # -export key="$(cat "$__object/parameter/key" 2>/dev/null \ +key="$(cat "$__object/parameter/key" 2>/dev/null \ || echo "$__object_id")" -export state="$(cat "$__object/parameter/state")" +state="$(cat "$__object/parameter/state")" file="$(cat "$__object/parameter/file")" @@ -30,14 +30,15 @@ if [ ! -f "$file" ]; then exit fi -export delimiter="$(cat "$__object/parameter/delimiter")" -export value="$(cat "$__object/parameter/value" 2>/dev/null \ +delimiter="$(cat "$__object/parameter/delimiter")" +value="$(cat "$__object/parameter/value" 2>/dev/null \ || echo "__CDIST_NOTSET__")" if [ -f "$__object/parameter/exact_delimiter" ]; then - export exact_delimiter=1 + exact_delimiter=1 else - export exact_delimiter=0 + exact_delimiter=0 fi +export key state delimiter value exact_delimiter awk -f - "$file" <<"AWK_EOF" BEGIN { From 34b571b7d9d98451910650339231c3c3c06cfb61 Mon Sep 17 00:00:00 2001 From: Jonas Weber Date: Thu, 4 Oct 2018 22:22:22 +0200 Subject: [PATCH 134/178] Fix SC2091 (execute result of condition) For some reason, the echo|grep pattern was enclosed four times in a $() operation. Nothing happened, since grep is always invoked with -q (quiet, nothing printed to stdout). --- cdist/conf/type/__file/gencode-local | 2 +- cdist/conf/type/__jail_freebsd10/gencode-local | 2 +- cdist/conf/type/__jail_freebsd9/gencode-local | 2 +- cdist/conf/type/__user/gencode-remote | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 3f60b4b3..fb9f9a92 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -84,7 +84,7 @@ DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix - if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index b2016f86..f163cad3 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -44,7 +44,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index 1ab7ff1a..bbdc9fcc 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -40,7 +40,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index 332e93f1..a1347a6f 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -60,7 +60,7 @@ if [ "$state" = "present" ]; then case "$property" in gid) - if $(echo "$new_value" | grep -q '^[0-9][0-9]*$'); then + if echo "$new_value" | grep -q '^[0-9][0-9]*$'; then field=4 else # We were passed a group name. Compare the gid in From 738f0538775ae182e10ae90c56a38af094ba7063 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 22:39:06 +0200 Subject: [PATCH 135/178] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 1405ad3c..151b11e5 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,9 @@ next: * Explorers and types: Fix SC2039: In POSIX sh, something is undefined (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2045: Iterating over ls output is fragile. Use globs. (shellcheck) (Thomas Eckert) * Explorers and types: Fix SC2148: Tips depend on target shell and yours is unknown. Add a shebang. (shellcheck) (Thomas Eckert) + * Explorers and types: Fix SC2069: To redirect stdout+stderr, 2>&1 must be last (or use '{ cmd > file; } 2>&1' to clarify). (shellcheck) (Jonas Weber) + * Explorers and types: Fix SC2034: foo appears unused. Verify it or export it. (shellcheck) (Jonas Weber) + * Explorers and types: Fix SC2215, SC2059, SC2155, SC2091. (shellcheck) (Jonas Weber) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From c01cf386608cbef77c53d6b2bf3c2f6d2b2be91d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 23:10:42 +0200 Subject: [PATCH 136/178] Fix SC2164 --- cdist/conf/explorer/disks | 2 +- cdist/conf/type/__link/explorer/state | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index 7a5c0da0..7c60b17a 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -1,3 +1,3 @@ #!/bin/sh -cd /dev +cd /dev || exit 0 echo sd? hd? vd? diff --git a/cdist/conf/type/__link/explorer/state b/cdist/conf/type/__link/explorer/state index b8d8fc2b..587835a2 100755 --- a/cdist/conf/type/__link/explorer/state +++ b/cdist/conf/type/__link/explorer/state @@ -32,7 +32,7 @@ destination_dir="${destination%/*}" case "$type" in symbolic) - cd "$destination_dir" + cd "$destination_dir" || exit 1 source_is=$(ls -l "$destination" | sed 's/.*-> //g') if [ -h "$destination" ]; then # ignore trailing slashes for comparison @@ -46,7 +46,7 @@ case "$type" in fi ;; hard) - cd "$destination_dir" + cd "$destination_dir" || exit 1 # check source relative to destination_dir if [ ! -e "$source" ]; then echo sourcemissing From d2f1d6185d7c9e46ae60e99cefc31f0b113ec148 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 10:29:47 +0200 Subject: [PATCH 137/178] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 151b11e5..d53edba2 100644 --- a/docs/changelog +++ b/docs/changelog @@ -14,6 +14,8 @@ next: * Explorers and types: Fix SC2069: To redirect stdout+stderr, 2>&1 must be last (or use '{ cmd > file; } 2>&1' to clarify). (shellcheck) (Jonas Weber) * Explorers and types: Fix SC2034: foo appears unused. Verify it or export it. (shellcheck) (Jonas Weber) * Explorers and types: Fix SC2215, SC2059, SC2155, SC2091. (shellcheck) (Jonas Weber) + * Explorers and types: Fix SC2086: Double quote to prevent globbing and word splitting. (shellcheck) (Jonas Weber) + * Explorers and types: Fix SC2164: Use cd ... || exit in case cd fails. (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 3b6d75966011a80a2d396933abe7d1816245c8e5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 4 Oct 2018 23:24:30 +0200 Subject: [PATCH 138/178] Disable SC2016 check. --- cdist/conf/type/__docker_compose/gencode-remote | 5 +++-- cdist/conf/type/__golang_from_vendor/manifest | 1 + cdist/conf/type/__package_emerge/explorer/pkg_version | 1 + cdist/conf/type/__rvm_gemset/explorer/state | 2 ++ cdist/conf/type/__staged_file/gencode-local | 3 +++ 5 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index 396c93ac..77fc2fdf 100755 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -24,8 +24,9 @@ state="$(cat "$__object/parameter/state")" if [ "${state}" = "present" ]; then # Download docker-compose file - echo 'curl -L "https://github.com/docker/compose/releases/download/'"${version}"'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' - echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' + #shellcheck disable=SC2016 + echo 'curl -L "https://github.com/docker/compose/releases/download/'"${version}"'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' + echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' # Change permissions echo 'chmod +x /usr/local/bin/docker-compose' fi diff --git a/cdist/conf/type/__golang_from_vendor/manifest b/cdist/conf/type/__golang_from_vendor/manifest index cf164524..ad39ddfb 100755 --- a/cdist/conf/type/__golang_from_vendor/manifest +++ b/cdist/conf/type/__golang_from_vendor/manifest @@ -1,3 +1,4 @@ #!/bin/sh -e +# shellcheck disable=SC2016 __line go_in_path --line 'export PATH=/usr/local/go/bin:$PATH' --file /etc/profile diff --git a/cdist/conf/type/__package_emerge/explorer/pkg_version b/cdist/conf/type/__package_emerge/explorer/pkg_version index 7053eaff..d02b9d6b 100644 --- a/cdist/conf/type/__package_emerge/explorer/pkg_version +++ b/cdist/conf/type/__package_emerge/explorer/pkg_version @@ -32,4 +32,5 @@ else name="$__object_id" fi +# shellcheck disable=SC2016 equery -q l -F '$cp $fullversion' "$name" || true diff --git a/cdist/conf/type/__rvm_gemset/explorer/state b/cdist/conf/type/__rvm_gemset/explorer/state index d1462134..e300453b 100755 --- a/cdist/conf/type/__rvm_gemset/explorer/state +++ b/cdist/conf/type/__rvm_gemset/explorer/state @@ -25,7 +25,9 @@ if [ ! -e "~$user/.rvm/scripts/rvm" ] ; then exit 0 fi +# shellcheck disable=SC2016 if su - "$user" -c 'source ~/.rvm/scripts/rvm; rvm list strings | grep -q "^$ruby\$"'; then + # shellcheck disable=SC2016 if su - "$user" -c 'source ~/.rvm/scripts/rvm; rvm use "$ruby" > /dev/null; rvm gemset list strings | cut -f 1 -d " " | grep -q "^$gemsetname\$"'; then echo "present" exit 0 diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local index 617ca23a..4edaba7f 100755 --- a/cdist/conf/type/__staged_file/gencode-local +++ b/cdist/conf/type/__staged_file/gencode-local @@ -62,7 +62,9 @@ fetch_file() { } fetch_and_prepare_file() { + # shellcheck disable=SC2016 printf 'tmpdir="$(mktemp -d --tmpdir="/tmp" "%s")"\n' "${__type##*/}.XXXXXXXXXX" + # shellcheck disable=SC2016 printf 'cd "$tmpdir"\n' # shellcheck disable=SC2059 printf "$fetch_command > \"%s\"\n" "$source" "$source_file_name" @@ -70,6 +72,7 @@ fetch_and_prepare_file() { # shellcheck disable=SC2059 printf "$prepare_command > \"%s\"\n" "$source_file_name" "$stage_file" printf 'cd - >/dev/null\n' + # shellcheck disable=SC2016 printf 'rm -rf "$tmpdir"\n' } From 5f8c631e344ed8b54c670748a08615161aea2ada Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 11:26:47 +0200 Subject: [PATCH 139/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index d53edba2..b5c218bc 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * Explorers and types: Fix SC2215, SC2059, SC2155, SC2091. (shellcheck) (Jonas Weber) * Explorers and types: Fix SC2086: Double quote to prevent globbing and word splitting. (shellcheck) (Jonas Weber) * Explorers and types: Fix SC2164: Use cd ... || exit in case cd fails. (shellcheck) (Darko Poljak) + * Explorers and types: Disable SC2016 where it is intended: Expressions don't expand in single quotes, use double quotes for that. (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From a601fa91d9c2eecc75bb476f81ccfb46e7d6d2c5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 11:34:10 +0200 Subject: [PATCH 140/178] Fix SC2116. --- cdist/conf/type/__hostname/gencode-remote | 2 +- cdist/conf/type/__hostname/manifest | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 3f784188..b729b667 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -22,7 +22,7 @@ if [ -f "$__object/parameter/name" ]; then name_should="$(cat "$__object/parameter/name")" else - name_should="$(echo "${__target_host%%.*}")" + name_should="${__target_host%%.*}" fi os=$(cat "$__global/explorer/os") diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index 74664aa7..c03b2eac 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -25,10 +25,10 @@ if [ -f "$__object/parameter/name" ]; then else case "$os" in openbsd) - name_should="$(echo "${__target_host}")" + name_should="${__target_host}" ;; *) - name_should="$(echo "${__target_host%%.*}")" + name_should="${__target_host%%.*}" ;; esac fi From b25aff17c6cbeaf26cec3ce17f250bdacdd38df2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 11:49:28 +0200 Subject: [PATCH 141/178] Disable SC2039 where appropriate. --- cdist/conf/type/__package_pkg_freebsd/gencode-remote | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 6c6e15b1..fd02d939 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -36,6 +36,7 @@ assert () # If condition false, if [ ! "$1" ] then echo "Assertion failed: \"$1\"" + # shellcheck disable=SC2039 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi @@ -95,6 +96,7 @@ if [ -n "$curr_version" ]; then # PKG *is* installed exit 0 else # Current version is wrong, fix #updatepkg "$name" "$version" + # shellcheck disable=SC2039 assert "! ${version} = ${curr_version}" $LINENO cmd="${rm_cmd} ${name}-${curr_version}" execcmd "remove" "${cmd}" From ed542cc7cb3f8e02a2dea1388e8434d0987eeac4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 11:57:31 +0200 Subject: [PATCH 142/178] Fix SC2046. --- cdist/conf/type/__jail_freebsd10/gencode-remote | 2 +- cdist/conf/type/__jail_freebsd9/gencode-remote | 2 +- cdist/conf/type/__package_emerge/gencode-remote | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 5be04bc6..84b2bae8 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -83,7 +83,7 @@ present="$(cat "$__object/explorer/present")" status="$(cat "$__object/explorer/status")" # Handle ip="addr, addr" format -if [ $(expr "${ip}" : ".*, .*") -gt "0" ]; then +if [ "$(expr "${ip}" : ".*, .*")" -gt "0" ]; then SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index 177bdf74..5b103ebc 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -86,7 +86,7 @@ present="$(cat "$__object/explorer/present")" status="$(cat "$__object/explorer/status")" # Handle ip="iface|addr, iface|addr" format -if [ $(expr "${ip}" : ".*|.*") -gt "0" ]; then +if [ "$(expr "${ip}" : ".*|.*")" -gt "0" ]; then # If we have multiple IPs defined, $interface doesn't make sense because ip="iface|addr, iface|addr" implies it interface="" SAVE_IFS="$IFS" diff --git a/cdist/conf/type/__package_emerge/gencode-remote b/cdist/conf/type/__package_emerge/gencode-remote index 8abcb5b1..c8a79c1f 100755 --- a/cdist/conf/type/__package_emerge/gencode-remote +++ b/cdist/conf/type/__package_emerge/gencode-remote @@ -38,11 +38,11 @@ fi pkg_version="$(cat "$__object/explorer/pkg_version")" if [ -z "$pkg_version" ]; then state_is="absent" -elif [ -z "$version" ] && [ $(echo "$pkg_version" | wc -l) -gt 1 ]; then +elif [ -z "$version" ] && [ "$(echo "$pkg_version" | wc -l)" -gt 1 ]; then echo "Package name is not unique! The following packages are installed:" echo "$pkg_version" exit 1 -elif [ -n "$version" ] && [ $(echo "$pkg_version" | cut -d " " -f 1 | sort | uniq | wc -l) -gt 1 ]; then +elif [ -n "$version" ] && [ "$(echo "$pkg_version" | cut -d " " -f 1 | sort | uniq | wc -l)" -gt 1 ]; then echo "Package name is not unique! The following packages are installed:" echo "$pkg_version" exit 1 From 28889ef38a08f53b193a7cc561372e8c03175853 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:02:10 +0200 Subject: [PATCH 143/178] Disable SC2028 where intended. --- cdist/conf/type/__hostname/gencode-remote | 2 ++ cdist/conf/type/__jail_freebsd10/gencode-remote | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index b729b667..fd9e7c98 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -59,6 +59,7 @@ echo changed >> "$__messages_out" # Use the good old way to set the hostname even on machines running systemd. case "$os" in archlinux|debian|ubuntu|devuan|centos|coreos) + # shellcheck disable=SC2028 echo "printf '%s\n' '$name_should' > /etc/hostname" echo "hostname -F /etc/hostname" ;; @@ -67,6 +68,7 @@ case "$os" in ;; suse) echo "hostname '$name_should'" + # shellcheck disable=SC2028 echo "printf '%s\n' '$name_should' > /etc/HOSTNAME" ;; esac diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 84b2bae8..81912050 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -298,7 +298,7 @@ END fi EOF - + # shellcheck disable=SC2028 echo "printf \"%s\\n%s\n%s\n\" \"\$jailheader\" \"\$jaildata\" \"\$jailtrailer\" >>\"\$jailfile\"" # Add $name to jail_list if $onboot=yes From 73acb35a8acf1f7f235e6cda40dc43563b261b00 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:04:45 +0200 Subject: [PATCH 144/178] Fix SC2196. --- cdist/conf/type/__package_luarocks/explorer/pkg_status | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_luarocks/explorer/pkg_status b/cdist/conf/type/__package_luarocks/explorer/pkg_status index 3eb73298..e83e8ce6 100755 --- a/cdist/conf/type/__package_luarocks/explorer/pkg_status +++ b/cdist/conf/type/__package_luarocks/explorer/pkg_status @@ -28,4 +28,4 @@ else fi # Accept luarocks failing if package is not known/installed -luarocks list "$name" | egrep -A1 "^$name$" || exit 0 +luarocks list "$name" | grep -E -A1 "^$name$" || exit 0 From 73d0a5bbbed2e777ea721e47b9e5dd6a2bc7859c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:14:51 +0200 Subject: [PATCH 145/178] Fix SC2235. --- cdist/conf/type/__package_emerge/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_emerge/gencode-remote b/cdist/conf/type/__package_emerge/gencode-remote index c8a79c1f..48462bde 100755 --- a/cdist/conf/type/__package_emerge/gencode-remote +++ b/cdist/conf/type/__package_emerge/gencode-remote @@ -57,7 +57,7 @@ fi # Exit if nothing is needed to be done -[ "$state_is" = "$state_should" ] && ( [ -z "$version" ] || [ "$installed_version" = "$version" ] ) && exit 0 +[ "$state_is" = "$state_should" ] && { [ -z "$version" ] || [ "$installed_version" = "$version" ]; } && exit 0 [ "$state_should" = "absent" ] && [ ! -z "$version" ] && [ "$installed_version" != "$version" ] && exit 0 From 2f7b8e6db9ed4900c3fdc9f3dac26654e706b8ae Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:16:27 +0200 Subject: [PATCH 146/178] Fix SC2153. --- cdist/conf/type/__pf_ruleset/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index b4bded98..b9de1bce 100755 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -54,7 +54,7 @@ case $uname in currentSum=\$(cksum -o 1 ${source} | cut -d= -f2 | sed 's/ //g') ;; *) - echo "Sorry, I do not know how to find a cksum on ${UNAME}." >&2 + echo "Sorry, I do not know how to find a cksum on ${uname}." >&2 exit 1 ;; esac From e46420e7f57560a5b833d42dfcf10c40549f83d9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:19:16 +0200 Subject: [PATCH 147/178] Fix SC1090. --- cdist/conf/type/__docker/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index 35760cef..8f26feec 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -24,6 +24,7 @@ state=$(cat "$__object/parameter/state") case "$os" in centos) + # shellcheck source=/dev/null if (. "$__global/explorer/os_release" && [ "${VERSION_ID}" = "7" ]); then __yum_repo docker-ce-stable \ --name 'Docker CE Stable' \ From 7b17ad473303a48a70b1d959141bdf22b8747607 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:21:59 +0200 Subject: [PATCH 148/178] Fix SC2143. --- cdist/conf/type/__jail/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__jail/manifest b/cdist/conf/type/__jail/manifest index 222cb859..6fb4a306 100755 --- a/cdist/conf/type/__jail/manifest +++ b/cdist/conf/type/__jail/manifest @@ -44,7 +44,7 @@ for property in *; do done ver="$(cat "$__global/explorer/os_version")" -if [ -n "$(echo "$ver" | grep '^10\.' )" ]; then # Version is 10.x +if echo "$ver" | grep -q '^10\.'; then # Version is 10.x __jail_freebsd10 "$@" else __jail_freebsd9 "$@" From b120a813a2e2db27e29c8442fba9d0091f67a45c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:24:03 +0200 Subject: [PATCH 149/178] Disable SC2030 where appropriate. --- cdist/conf/type/__consul_template_template/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index 7834ae9c..4ba0329f 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -52,6 +52,7 @@ for param in *; do --owner root --group root --mode 640 \ --source "$source" \ --state "$state" + # shellcheck disable=SC2030 export require="__file${destination}" printf ' source = "%s"\n' "$destination" From 8cea8e67aaa71d911b70326efd7c8e61103b1502 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:25:40 +0200 Subject: [PATCH 150/178] Disable SC2031 where appropriate. --- cdist/conf/type/__consul_template_template/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index 4ba0329f..2fd0a46c 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -17,6 +17,7 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# shellcheck disable=SC2031 name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")" state="$(cat "$__object/parameter/state")" From 9b567ed8f7030de20066f65f0d78e7bc323e6af1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:27:43 +0200 Subject: [PATCH 151/178] Fix SC2230. --- cdist/conf/type/__dot_file/explorer/home | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__dot_file/explorer/home b/cdist/conf/type/__dot_file/explorer/home index 132cfc71..08d941bf 100755 --- a/cdist/conf/type/__dot_file/explorer/home +++ b/cdist/conf/type/__dot_file/explorer/home @@ -19,7 +19,7 @@ set -eu user="$(cat "${__object}/parameter/user")" -if which getent >/dev/null 2>&1; then +if command -v getent >/dev/null 2>&1; then line=$(getent passwd "${user}") else line=$(grep "^${user}:" /etc/passwd) From 596f3cf227c3e0ee219caf00a1973baf0d68db87 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:30:52 +0200 Subject: [PATCH 152/178] Disable SC2119,SC2120 where appropriate. --- cdist/conf/type/__jail_freebsd10/gencode-remote | 2 ++ cdist/conf/type/__jail_freebsd9/gencode-remote | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 81912050..d01b30c6 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -111,6 +111,7 @@ startJail() { fi } +# shellcheck disable=SC2120 deleteJail() { # Unmount the jail's mountpoints if necessary cat < Date: Fri, 5 Oct 2018 12:32:56 +0200 Subject: [PATCH 153/178] Fix SC2145. --- .../conf/type/__install_partition_msdos_apply/gencode-remote | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index 644de33c..d699d8ad 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -21,11 +21,11 @@ #set -x die() { - echo "[__install_partition_msdos_apply] $@" >&2 + echo "[__install_partition_msdos_apply] $*" >&2 exit 1 } debug() { - #echo "[__install_partition_msdos_apply] $@" >&2 + #echo "[__install_partition_msdos_apply] $*" >&2 : } From 644ec85d801e17f38b8917b2822e611db97a1979 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:50:01 +0200 Subject: [PATCH 154/178] Fix SC2162. --- cdist/conf/type/__rsync/gencode-local | 2 +- cdist/conf/type/__ssh_authorized_key/gencode-remote | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index c7196175..e36ded2f 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -29,7 +29,7 @@ fi set -- if [ -f "$__object/parameter/rsync-opts" ]; then - while read opts; do + while read -r opts; do set -- "$@" "--$opts" done < "$__object/parameter/rsync-opts" fi diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 325854c2..82c90d61 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -78,7 +78,7 @@ if [ -s "$__object/explorer/entry" ]; then # Note that the files have to be sorted for comparison with `comm`. sort "$__object/explorer/entry" > "$__object/files/is" comm -13 "$__object/files/should" "$__object/files/is" | { - while read entry; do + while read -r entry; do remove_line "$file" "$entry" done } From a369acea72d173aa7f8bb2793ece4b8a91839caa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 12:52:44 +0200 Subject: [PATCH 155/178] Fix SC2021. --- cdist/conf/type/__postgres_role/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 14240992..f977e73e 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -34,7 +34,7 @@ case "$state_should" in if [ ! -f "$__object/parameter/$boolean" ]; then boolean="no${boolean}" fi - upper=$(echo $boolean | tr '[a-z]' '[A-Z]') + upper=$(echo $boolean | tr '[:lower:]' '[:upper:]') booleans="$booleans $upper" done From bf270b6f9c32ab345ed3ee30dc70aeb7a1ca8d81 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 13:05:01 +0200 Subject: [PATCH 156/178] Fix SC2044. --- cdist/conf/type/__install_generate_fstab/gencode-local | 4 +++- cdist/conf/type/__install_mount/gencode-remote | 4 +++- .../conf/type/__install_partition_msdos_apply/gencode-remote | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__install_generate_fstab/gencode-local b/cdist/conf/type/__install_generate_fstab/gencode-local index aca42761..80455aaa 100755 --- a/cdist/conf/type/__install_generate_fstab/gencode-local +++ b/cdist/conf/type/__install_generate_fstab/gencode-local @@ -25,7 +25,9 @@ mkdir "$__object/files" # get current UUID's from target_host $__remote_exec "$__target_host" blkid > "$__object/files/blkid" -for object in $(find "$__global/object/__install_mount" -type d -name "$__cdist_object_marker"); do +find "$__global/object/__install_mount" -type d -name "$__cdist_object_marker" | +while IFS= read -r object +do device="$(cat "$object/parameter/device")" dir="$(cat "$object/parameter/dir")" type="$(cat "$object/parameter/type")" diff --git a/cdist/conf/type/__install_mount/gencode-remote b/cdist/conf/type/__install_mount/gencode-remote index ce96279a..4415f0ff 100755 --- a/cdist/conf/type/__install_mount/gencode-remote +++ b/cdist/conf/type/__install_mount/gencode-remote @@ -20,7 +20,9 @@ get_type_from_mkfs() { _device="$1" - for mkfs_object in $(find "$__global/object/__install_mkfs" -type d -name "$__cdist_object_marker"); do + find "$__global/object/__install_mkfs" -type d -name "$__cdist_object_marker" | + while IFS= read -r mkfs_object + do mkfs_device="$(cat "$mkfs_object/parameter/device")" if [ "$_device" = "$mkfs_device" ]; then cat "$mkfs_object/parameter/type" diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index 644de33c..69be8daf 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -59,7 +59,9 @@ size_to_mb() { get_objects() { objects_file=$(mktemp) - for object in $(find "$__global/object/__install_partition_msdos" -type d -name "$__cdist_object_marker"); do + find "$__global/object/__install_partition_msdos" -type d -name "$__cdist_object_marker" | + while IFS= read -r object + do object_device="$(cat "$object/parameter/device")" object_minor="$(cat "$object/parameter/minor")" echo "$object_device $object_minor $object" >> "$objects_file" From 9a61aab631a9a0a8425d20affe603f1ab2d194ae Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 14:59:35 +0200 Subject: [PATCH 157/178] ++ --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index b5c218bc..419a3003 100644 --- a/docs/changelog +++ b/docs/changelog @@ -17,6 +17,7 @@ next: * Explorers and types: Fix SC2086: Double quote to prevent globbing and word splitting. (shellcheck) (Jonas Weber) * Explorers and types: Fix SC2164: Use cd ... || exit in case cd fails. (shellcheck) (Darko Poljak) * Explorers and types: Disable SC2016 where it is intended: Expressions don't expand in single quotes, use double quotes for that. (shellcheck) (Darko Poljak) + * Explorers and types: Fix SC2044: For loops over find output are fragile. Use find -exec or a while read loop. (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 0d80b4473b58623eaea63e115310e03201dd11fc Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 15:04:22 +0200 Subject: [PATCH 158/178] Disable SC2031 for the single incident. --- cdist/conf/type/__consul_template_template/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index 2fd0a46c..506bc528 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -17,7 +17,6 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# shellcheck disable=SC2031 name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")" state="$(cat "$__object/parameter/state")" @@ -37,6 +36,7 @@ if [ ! -f "$__object/parameter/source" ] && [ ! -f "$__object/parameter/source-f fi # Generate hcl config file +# shellcheck disable=SC2031 ( printf 'template {\n' cd "$__object/parameter/" From 02f39bcf6f32647e7ee2c4cbd7781439c3567f16 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 15:08:25 +0200 Subject: [PATCH 159/178] Fix SC2028. --- cdist/conf/type/__hostname/gencode-remote | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index fd9e7c98..fc50b651 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -59,8 +59,7 @@ echo changed >> "$__messages_out" # Use the good old way to set the hostname even on machines running systemd. case "$os" in archlinux|debian|ubuntu|devuan|centos|coreos) - # shellcheck disable=SC2028 - echo "printf '%s\n' '$name_should' > /etc/hostname" + printf "printf '%s\\n' '$name_should' > /etc/hostname\\n" echo "hostname -F /etc/hostname" ;; freebsd|openbsd) @@ -68,8 +67,7 @@ case "$os" in ;; suse) echo "hostname '$name_should'" - # shellcheck disable=SC2028 - echo "printf '%s\n' '$name_should' > /etc/HOSTNAME" + printf "printf '%s\\n' '$name_should' > /etc/HOSTNAME\\n" ;; esac From b284bafbee38be49b57b2b6caf8af8d6df4be2c9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 15:14:03 +0200 Subject: [PATCH 160/178] grep directly on the explorer file --- cdist/conf/type/__jail/manifest | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cdist/conf/type/__jail/manifest b/cdist/conf/type/__jail/manifest index 6fb4a306..adf9bd99 100755 --- a/cdist/conf/type/__jail/manifest +++ b/cdist/conf/type/__jail/manifest @@ -43,8 +43,7 @@ for property in *; do set -- "$@" "--$property" "$(cat "$property")" done -ver="$(cat "$__global/explorer/os_version")" -if echo "$ver" | grep -q '^10\.'; then # Version is 10.x +if grep -q '^10\.' "$(cat "$__global/explorer/os_version")" ; then # Version is 10.x __jail_freebsd10 "$@" else __jail_freebsd9 "$@" From 68fad03a6b98c48abf6cca05488998d7df872c7b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 5 Oct 2018 15:18:59 +0200 Subject: [PATCH 161/178] Fix SC2120, SC2119. --- cdist/conf/type/__jail_freebsd10/gencode-remote | 6 ++---- cdist/conf/type/__jail_freebsd9/gencode-remote | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index d01b30c6..230370c2 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -111,7 +111,6 @@ startJail() { fi } -# shellcheck disable=SC2120 deleteJail() { # Unmount the jail's mountpoints if necessary cat <=1 rw mount is mounted still for DIR in "${output}"; do - umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print $3}')" + umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi output="\$(mount | grep "\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still - umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print $3}')" + umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print \$3}')" fi EOF # Remove the jail's rw mountpoints @@ -348,7 +347,6 @@ if [ "$present" = "EXISTS" ]; then # The jail currently exists exit 0 else # The jail is not supposed to exist stopJail - # shellcheck disable=SC2119 deleteJail exit 0 fi diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index e89672c2..13c237a9 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -116,7 +116,6 @@ startJail() { fi } -# shellcheck disable=SC2120 deleteJail() { # Unmount the jail's mountpoints if necessary cat <=1 rw mount is mounted still for DIR in "${output}"; do - umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print $3}')" + umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi output="\$(mount | grep "\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still - umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print $3}')" + umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print \$3}')" fi EOF # Remove the jail's rw mountpoints @@ -343,7 +342,6 @@ if [ "$present" = "EXISTS" ]; then # The jail currently exists exit 0 else # The jail is not supposed to exist stopJail - # shellcheck disable=SC2119 deleteJail exit 0 fi From f039b0e6e5e4d90793061cc2a6dc73883367d767 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 08:36:06 +0200 Subject: [PATCH 162/178] Fix SC2030,SC2031. --- cdist/conf/type/__consul_template_template/manifest | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index 506bc528..1eae1fad 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -35,9 +35,13 @@ if [ ! -f "$__object/parameter/source" ] && [ ! -f "$__object/parameter/source-f exit 1 fi +if [ -f "$__object/parameter/source-file" ]; then + destination="${template_dir}/${name}" + require="__file${destination}" +fi + # Generate hcl config file -# shellcheck disable=SC2031 -( +{ printf 'template {\n' cd "$__object/parameter/" for param in *; do @@ -47,14 +51,11 @@ for param in *; do if [ "$source" = "-" ]; then source="$__object/stdin" fi - destination="${template_dir}/${name}" require="__directory${template_dir}" \ __file "$destination" \ --owner root --group root --mode 640 \ --source "$source" \ --state "$state" - # shellcheck disable=SC2030 - export require="__file${destination}" printf ' source = "%s"\n' "$destination" ;; @@ -68,7 +69,7 @@ for param in *; do esac done printf '}\n' -) | \ +} | \ require="$require __directory${conf_dir}" \ __config_file "${conf_dir}/${conf_file}" \ --owner root --group root --mode 640 \ From 877b4d2748018096fded1c9e81ee73a2257c53b5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 15:15:53 +0200 Subject: [PATCH 163/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 419a3003..bc51c81b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -18,6 +18,7 @@ next: * Explorers and types: Fix SC2164: Use cd ... || exit in case cd fails. (shellcheck) (Darko Poljak) * Explorers and types: Disable SC2016 where it is intended: Expressions don't expand in single quotes, use double quotes for that. (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2044: For loops over find output are fragile. Use find -exec or a while read loop. (shellcheck) (Darko Poljak) + * Explorers and types: Fix SC1090,SC2021,SC2028,SC2030,SC2031,SC2039,SC2046,SC2116,SC2119,SC2120,SC2143,SC2145,SC2153,SC2162,SC2196,SC2230,SC2235. (shellcheck) (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 8436f872a9cc32162e8991751558ecfcbcc7e967 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 14:57:55 +0200 Subject: [PATCH 164/178] Add shellcheck makefile target. shellcheck when doing release. --- Makefile | 20 ++++++++++++++++++++ bin/build-helper | 25 ++++++++++++++++++++++++- bin/build-helper.freebsd | 25 ++++++++++++++++++++++++- 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 3a1819be..4a756808 100644 --- a/Makefile +++ b/Makefile @@ -39,6 +39,9 @@ PYTHON_VERSION=cdist/version.py SPHINXM=make -C $(DOCS_SRC_DIR) man SPHINXH=make -C $(DOCS_SRC_DIR) html SPHINXC=make -C $(DOCS_SRC_DIR) clean + +SHELLCHECKCMD=shellcheck -s sh -f gcc -x +SHELLCHECK_SKIP=grep -v ': __.*is referenced but not assigned.*\[SC2154\]' ################################################################################ # Manpages # @@ -253,3 +256,20 @@ test-remote: pep8: $(helper) $@ + +shellcheck-global-explorers: + @find cdist/conf/explorer -type f -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) +shellcheck-manifests: + @find cdist/conf/type -type f -name manifest -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + +shellcheck-local-gencodes: + @find cdist/conf/type -type f -name gencode-local -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + +shellcheck-remote-gencodes: + @find cdist/conf/type -type f -name gencode-remote -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + +shellcheck-gencodes: shellcheck-local-gencodes shellcheck-remote-gencodes + +shellcheck-types: shellcheck-manifests shellcheck-gencodes + +shellcheck: shellcheck-global-explorers shellcheck-types diff --git a/bin/build-helper b/bin/build-helper index ef15b6c2..02fa67d6 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -250,6 +250,7 @@ eof "$0" check-date "$0" check-unittest "$0" check-pep8 + "$0" shellcheck # Generate version file to be included in packaging "$0" target-version @@ -370,7 +371,29 @@ eof check-pep8) "$0" pep8 - echo "Please review pep8 report." + printf "\\nPlease review pep8 report.\\n" + while true + do + echo "Continue (yes/no)?" + any= + read any + case "$any" in + yes) + break + ;; + no) + exit 1 + ;; + *) + echo "Please answer with 'yes' or 'no' explicitly." + ;; + esac + done + ;; + + shellcheck) + make helper=${helper} WEBDIR=${WEBDIR} shellcheck + printf "\\nPlease review shellcheck report.\\n" while true do echo "Continue (yes/no)?" diff --git a/bin/build-helper.freebsd b/bin/build-helper.freebsd index e49b2f7a..081feb54 100755 --- a/bin/build-helper.freebsd +++ b/bin/build-helper.freebsd @@ -285,6 +285,7 @@ eof "$0" check-date "$0" check-unittest "$0" check-pep8 + "$0" shellcheck # Generate version file to be included in packaging "$0" target-version @@ -432,7 +433,29 @@ eof check-pep8) "$0" pep8 - echo "Please review pep8 report." + printf "\\nPlease review pep8 report.\\n" + while true + do + echo "Continue (yes/no)?" + any= + read any + case "$any" in + yes) + break + ;; + no) + exit 1 + ;; + *) + echo "Please answer with 'yes' or 'no' explicitly." + ;; + esac + done + ;; + + shellcheck) + make helper=${helper} WEBDIR=${WEBDIR} shellcheck + printf "\\nPlease review shellcheck report.\\n" while true do echo "Continue (yes/no)?" From 140518f1b0dddb1778d4e4c9ce3903f893a5a53e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 15:00:31 +0200 Subject: [PATCH 165/178] Add comment for SHELLCHECK_SKIP. --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 4a756808..56229496 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,8 @@ SPHINXH=make -C $(DOCS_SRC_DIR) html SPHINXC=make -C $(DOCS_SRC_DIR) clean SHELLCHECKCMD=shellcheck -s sh -f gcc -x +# Skip SC2154 for variables starting with __ since such variables are cdist +# environment variables. SHELLCHECK_SKIP=grep -v ': __.*is referenced but not assigned.*\[SC2154\]' ################################################################################ # Manpages From edaf4b46a521e2073de829ce5989127451b216ee Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 22:53:07 +0200 Subject: [PATCH 166/178] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bc51c81b..d7d31f52 100644 --- a/docs/changelog +++ b/docs/changelog @@ -19,6 +19,7 @@ next: * Explorers and types: Disable SC2016 where it is intended: Expressions don't expand in single quotes, use double quotes for that. (shellcheck) (Darko Poljak) * Explorers and types: Fix SC2044: For loops over find output are fragile. Use find -exec or a while read loop. (shellcheck) (Darko Poljak) * Explorers and types: Fix SC1090,SC2021,SC2028,SC2030,SC2031,SC2039,SC2046,SC2116,SC2119,SC2120,SC2143,SC2145,SC2153,SC2162,SC2196,SC2230,SC2235. (shellcheck) (Darko Poljak) + * Build: Add shellcheck makefile target and check when doing release (Darko Poljak) 4.10.3: 2018-09-23 * New global explorer: os_release (Ľubomír Kučera) From 3be1c4ad6f4a5f97d346f0a2a54b5526d8deb0ca Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 10:21:08 +0200 Subject: [PATCH 167/178] If nothing found shellcheck targets should succeed. --- Makefile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 56229496..e9cacced 100644 --- a/Makefile +++ b/Makefile @@ -260,15 +260,16 @@ pep8: $(helper) $@ shellcheck-global-explorers: - @find cdist/conf/explorer -type f -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + @find cdist/conf/explorer -type f -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 + shellcheck-manifests: - @find cdist/conf/type -type f -name manifest -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + @find cdist/conf/type -type f -name manifest -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 shellcheck-local-gencodes: - @find cdist/conf/type -type f -name gencode-local -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + @find cdist/conf/type -type f -name gencode-local -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 shellcheck-remote-gencodes: - @find cdist/conf/type -type f -name gencode-remote -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) + @find cdist/conf/type -type f -name gencode-remote -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 shellcheck-gencodes: shellcheck-local-gencodes shellcheck-remote-gencodes From 83806e767a834ce80cc35f756090356fe276a205 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 13:25:16 +0200 Subject: [PATCH 168/178] Disable SC1091 where appropriate. --- cdist/conf/explorer/lsb_codename | 1 + cdist/conf/explorer/lsb_description | 1 + cdist/conf/explorer/lsb_id | 1 + cdist/conf/explorer/lsb_release | 1 + 4 files changed, 4 insertions(+) diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index bfabd444..26bb8e3d 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -22,6 +22,7 @@ set +e case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") ;; *) diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index 441fdbf3..b1009627 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -22,6 +22,7 @@ set +e case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") ;; *) diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 1a12b610..82ff9977 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -22,6 +22,7 @@ set +e case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") ;; *) diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 25eb95b1..5ebfff1a 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -22,6 +22,7 @@ set +e case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") ;; *) From a9f27beb983b2f29e0af7b4b62434088556d292b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 13:41:09 +0200 Subject: [PATCH 169/178] Fix SC1117. --- cdist/conf/type/__cron/gencode-remote | 2 +- cdist/conf/type/__install_coreos/gencode-remote | 2 +- .../type/__install_reset_disk/gencode-remote | 2 +- cdist/conf/type/__jail_freebsd10/gencode-remote | 16 ++++++++-------- cdist/conf/type/__jail_freebsd9/gencode-remote | 12 ++++++------ cdist/conf/type/__mount/gencode-remote | 2 +- .../type/__package_pkgng_freebsd/gencode-remote | 2 +- cdist/conf/type/__rvm/gencode-remote | 2 +- .../type/__ssh_authorized_key/gencode-remote | 2 +- cdist/conf/type/__staged_file/gencode-local | 4 ++-- 10 files changed, 23 insertions(+), 23 deletions(-) diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote index f58896af..ee626485 100755 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -58,7 +58,7 @@ state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" # These are the old markers prefix="#cdist:__cron/$__object_id" suffix="#/cdist:__cron/$__object_id" -filter="^# DO NOT EDIT THIS FILE|^# \(.* installed on |^# \(Cron version V|^# \(Cronie version .\..\)$" +filter="^# DO NOT EDIT THIS FILE|^# \\(.* installed on |^# \\(Cron version V|^# \\(Cronie version .\\..\\)$" cat << DONE crontab -u $user -l 2>/dev/null | grep -v -E "$filter" | awk -v prefix="$prefix" -v suffix="$suffix" ' { diff --git a/cdist/conf/type/__install_coreos/gencode-remote b/cdist/conf/type/__install_coreos/gencode-remote index fbe86479..f550b5a5 100755 --- a/cdist/conf/type/__install_coreos/gencode-remote +++ b/cdist/conf/type/__install_coreos/gencode-remote @@ -12,7 +12,7 @@ eof coreos-install -d "${device}" \ \$(if [ -s "\${ignition_file}" ]; then - printf -- "-i \${ignition_file}\n" + printf -- "-i \${ignition_file}\\n" fi) rm "\${ignition_file}" diff --git a/cdist/conf/type/__install_reset_disk/gencode-remote b/cdist/conf/type/__install_reset_disk/gencode-remote index 947dd472..ac9ae6cf 100755 --- a/cdist/conf/type/__install_reset_disk/gencode-remote +++ b/cdist/conf/type/__install_reset_disk/gencode-remote @@ -67,5 +67,5 @@ fi # erase partition table dd if=/dev/zero of=$disk bs=512 count=1 -printf 'w\n' | fdisk -u -c $disk || true +printf 'w\\n' | fdisk -u -c $disk || true DONE diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 230370c2..a5e097f9 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -45,7 +45,7 @@ else # when $state=present, it's required. Enforce this. if [ "$state" = "present" ]; then exec >&2 - echo "If --state is 'present,' --ip must be given\!" + echo "If --state is 'present,' --ip must be given\\!" exit 1 fi fi @@ -114,17 +114,17 @@ startJail() { deleteJail() { # Unmount the jail's mountpoints if necessary cat <=1 rw mount is mounted still for DIR in "${output}"; do umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi - output="\$(mount | grep "\/${name} (")" || true + output="\$(mount | grep "\\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print \$3}')" fi @@ -275,9 +275,9 @@ cat <&1 >/dev/null # Close the FD==fail... @@ -290,7 +290,7 @@ add include \\\$devfsrules_unhide_basic add include \\\$devfsrules_unhide_login END fi - devfsruleset_num=\$(grep "\[${devfsruleset}=" /etc/devfs.rules | sed -n 's/\[.*=\([0-9]*\)\]/\1/pg') + devfsruleset_num=\$(grep "\\[${devfsruleset}=" /etc/devfs.rules | sed -n 's/\\[.*=\\([0-9]*\\)\\]/\\1/pg') if [ -n "\$devfsruleset_num" ]; then jaildata="\$jaildata devfs_ruleset=\"\${devfsruleset_num}\";" @@ -299,7 +299,7 @@ END EOF # shellcheck disable=SC2028 - echo "printf \"%s\\n%s\n%s\n\" \"\$jailheader\" \"\$jaildata\" \"\$jailtrailer\" >>\"\$jailfile\"" + echo "printf \"%s\\n%s\\n%s\\n\" \"\$jailheader\" \"\$jaildata\" \"\$jailtrailer\" >>\"\$jailfile\"" # Add $name to jail_list if $onboot=yes if [ "$onboot" = "yes" ]; then diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index 13c237a9..7f3828b2 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -45,7 +45,7 @@ else # when $state=present, it's required. Enforce this. if [ "$state" = "present" ]; then exec >&2 - echo "If --state is 'present,' --ip must be given\!" + echo "If --state is 'present,' --ip must be given\\!" exit 1 fi fi @@ -119,17 +119,17 @@ startJail() { deleteJail() { # Unmount the jail's mountpoints if necessary cat <=1 rw mount is mounted still for DIR in "${output}"; do umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi - output="\$(mount | grep "\/${name} (")" || true + output="\$(mount | grep "\\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print \$3}')" fi @@ -279,9 +279,9 @@ END if [ ! -f /etc/devfs.rules ]; then touch /etc/devfs.rules fi - if [ -z "\$(grep '\[jailrules=' /etc/devfs.rules)" ]; then # The default ruleset doesn't exist + if [ -z "\$(grep '\\[jailrules=' /etc/devfs.rules)" ]; then # The default ruleset doesn't exist # Get the highest-numbered ruleset - highest="\$(sed -n 's/\[.*=\([0-9]*\)\]/\1/pg' /etc/devfs.rules | sort -u | tail -n 1)" || true + highest="\$(sed -n 's/\\[.*=\\([0-9]*\\)\\]/\\1/pg' /etc/devfs.rules | sort -u | tail -n 1)" || true # increment by 1 let num="\${highest}+1" 2>&- >&- # add default ruleset diff --git a/cdist/conf/type/__mount/gencode-remote b/cdist/conf/type/__mount/gencode-remote index 66d85f88..e8266a0b 100755 --- a/cdist/conf/type/__mount/gencode-remote +++ b/cdist/conf/type/__mount/gencode-remote @@ -39,7 +39,7 @@ case "$state_should" in printf ' -o %s' "$(cat "$__object/parameter/options")" fi printf ' %s' "$(cat "$__object/parameter/device")" - printf " %s\n" "$path" + printf " %s\\n" "$path" else # mount using existing fstab entry printf 'mount "%s"\n' "$path" diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index b59e3648..592d4d0f 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -95,7 +95,7 @@ if [ -n "$curr_version" ]; then # PKG *is* installed if [ "$upgrade" = "true" ]; then execcmd "upg" "${cmd}" else - printf "Version %s is already installed and pkg-ng can't upgrade directly to version %s.\nTo upgrade to the latest version, use the --upgrade flag.\n" "$curr_version" "$version" >&2 + printf "Version %s is already installed and pkg-ng can't upgrade directly to version %s.\\nTo upgrade to the latest version, use the --upgrade flag.\\n" "$curr_version" "$version" >&2 exit 1 fi # PKG is supposed to be installed to the latest version diff --git a/cdist/conf/type/__rvm/gencode-remote b/cdist/conf/type/__rvm/gencode-remote index 494c8fd8..993191c1 100755 --- a/cdist/conf/type/__rvm/gencode-remote +++ b/cdist/conf/type/__rvm/gencode-remote @@ -34,7 +34,7 @@ DONE absent) cat << DONE su - $user -c "rm -Rf \"\\\$HOME/.rvm\"; -sed '/rvm\/scripts\/rvm/d' \"\\\$HOME/.bashrc\" > \"\\\$HOME/.bashrc.cdist-tmp\" +sed '/rvm\\/scripts\\/rvm/d' \"\\\$HOME/.bashrc\" > \"\\\$HOME/.bashrc.cdist-tmp\" mv \"\\\$HOME/.bashrc.cdist-tmp\" \"\\\$HOME/.bashrc\"" DONE ;; diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 82c90d61..f37aa565 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -48,7 +48,7 @@ add_line() { line="$2" # escape single quotes line_sanitised=$(echo "$line" | sed -e "s/'/'\"'\"'/g") - printf '%s' "printf '%s\n' '$line_sanitised' >> $file" + printf '%s' "printf '%s\\n' '$line_sanitised' >> $file" } diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local index 4edaba7f..e78b50bd 100755 --- a/cdist/conf/type/__staged_file/gencode-local +++ b/cdist/conf/type/__staged_file/gencode-local @@ -67,10 +67,10 @@ fetch_and_prepare_file() { # shellcheck disable=SC2016 printf 'cd "$tmpdir"\n' # shellcheck disable=SC2059 - printf "$fetch_command > \"%s\"\n" "$source" "$source_file_name" + printf "$fetch_command > \"%s\"\\n" "$source" "$source_file_name" prepare_command="$(cat "$__object/parameter/prepare-command")" # shellcheck disable=SC2059 - printf "$prepare_command > \"%s\"\n" "$source_file_name" "$stage_file" + printf "$prepare_command > \"%s\"\\n" "$source_file_name" "$stage_file" printf 'cd - >/dev/null\n' # shellcheck disable=SC2016 printf 'rm -rf "$tmpdir"\n' From a78baccc827ece35501888cf83dab62144dac1ea Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 6 Oct 2018 13:59:47 +0200 Subject: [PATCH 170/178] Fix SC2154. Note: ignoring __ vars. --- cdist/conf/type/__jail_freebsd10/gencode-remote | 4 ++-- cdist/conf/type/__jail_freebsd9/gencode-remote | 4 ++-- cdist/conf/type/__package_opkg/gencode-remote | 2 +- cdist/conf/type/__pf_ruleset/gencode-local | 4 ++-- cdist/conf/type/__postgres_extension/gencode-remote | 2 +- cdist/conf/type/__process/gencode-remote | 1 + cdist/conf/type/__zypper_service/gencode-remote | 2 +- 7 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index a5e097f9..73518326 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -120,13 +120,13 @@ deleteJail() { fi output="\$(mount | grep "\\/rw\\/${name}\\/")" || true if [ -n "\${output}" ]; then # >=1 rw mount is mounted still - for DIR in "${output}"; do + for DIR in "\${output}"; do umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi output="\$(mount | grep "\\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still - umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print \$3}')" + umount -F "/etc/fstab.${name}" "\$(echo "\${output}" | awk '{print \$3}')" fi EOF # Remove the jail's rw mountpoints diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index 7f3828b2..be583e5d 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -125,13 +125,13 @@ deleteJail() { fi output="\$(mount | grep "\\/rw\\/${name}\\/")" || true if [ -n "\${output}" ]; then # >=1 rw mount is mounted still - for DIR in "${output}"; do + for DIR in "\${output}"; do umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi output="\$(mount | grep "\\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still - umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print \$3}')" + umount -F "/etc/fstab.${name}" "\$(echo "\${output}" | awk '{print \$3}')" fi EOF # Remove the jail's rw mountpoints diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index 7013ec9f..e94ff388 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -51,7 +51,7 @@ case "$state_should" in echo "opkg --verbosity=0 remove '$name'" ;; *) - echo "Unknown state: $state" >&2 + echo "Unknown state: ${state_should}" >&2 exit 1 ;; esac diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index b9de1bce..11bfb0b1 100755 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -69,10 +69,10 @@ fi if [ -n "${cksum}" ]; then if [ ! "\${currentSum}" = "${cksum}" ]; then - $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" + $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new" fi else # File just doesn't exist yet - $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" + $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new" fi EOF diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote index f7895103..627067c7 100755 --- a/cdist/conf/type/__postgres_extension/gencode-remote +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -33,7 +33,7 @@ case "$state_should" in echo "su - postgres -c 'psql -c \"$cmd\" \"$dbname\"'" ;; absent) - cmd="DROP EXTENSION IF EXISTS $extenstion" + cmd="DROP EXTENSION IF EXISTS $extension" echo "su - postgres -c 'psql -c \"$cmd\" \"$dbname\"'" ;; esac diff --git a/cdist/conf/type/__process/gencode-remote b/cdist/conf/type/__process/gencode-remote index fc491321..d14cf506 100755 --- a/cdist/conf/type/__process/gencode-remote +++ b/cdist/conf/type/__process/gencode-remote @@ -52,6 +52,7 @@ case "$state_should" in if [ -f "$__object/parameter/stop" ]; then cat "$__object/parameter/stop" else + # TODO: what should be runs variable? echo kill "${runs}" fi echo "stopped" >> "$__messages_out" diff --git a/cdist/conf/type/__zypper_service/gencode-remote b/cdist/conf/type/__zypper_service/gencode-remote index 59ffc942..4ccfe301 100755 --- a/cdist/conf/type/__zypper_service/gencode-remote +++ b/cdist/conf/type/__zypper_service/gencode-remote @@ -76,7 +76,7 @@ case "$state_should" in echo "zypper $zypper_def_opts ref" ;; absent) - echo "zypper $zypper_def_opts removeservice $service_id" + echo "zypper $zypper_def_opts removeservice $exp_id" echo "zypper $zypper_def_opts refs" echo "zypper $zypper_def_opts ref" ;; From e9b823bc390c30cb9294591c628c5353d194d0ef Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 09:59:01 +0200 Subject: [PATCH 171/178] Better fixes. --- cdist/conf/type/__cron/gencode-remote | 2 +- cdist/conf/type/__jail_freebsd10/gencode-remote | 2 +- cdist/conf/type/__jail_freebsd9/gencode-remote | 2 +- cdist/conf/type/__mount/gencode-remote | 2 +- cdist/conf/type/__package_pkgng_freebsd/gencode-remote | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote index ee626485..59398058 100755 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -58,7 +58,7 @@ state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" # These are the old markers prefix="#cdist:__cron/$__object_id" suffix="#/cdist:__cron/$__object_id" -filter="^# DO NOT EDIT THIS FILE|^# \\(.* installed on |^# \\(Cron version V|^# \\(Cronie version .\\..\\)$" +filter='^# DO NOT EDIT THIS FILE|^# \(.* installed on |^# \(Cron version V|^# \(Cronie version .\..\)$' cat << DONE crontab -u $user -l 2>/dev/null | grep -v -E "$filter" | awk -v prefix="$prefix" -v suffix="$suffix" ' { diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index 73518326..4f376c25 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -45,7 +45,7 @@ else # when $state=present, it's required. Enforce this. if [ "$state" = "present" ]; then exec >&2 - echo "If --state is 'present,' --ip must be given\\!" + printf 'If --state is "present", --ip must be given\!\n' exit 1 fi fi diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index be583e5d..68229d3e 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -45,7 +45,7 @@ else # when $state=present, it's required. Enforce this. if [ "$state" = "present" ]; then exec >&2 - echo "If --state is 'present,' --ip must be given\\!" + printf 'If --state is "present", --ip must be given\!\n' exit 1 fi fi diff --git a/cdist/conf/type/__mount/gencode-remote b/cdist/conf/type/__mount/gencode-remote index e8266a0b..b2096764 100755 --- a/cdist/conf/type/__mount/gencode-remote +++ b/cdist/conf/type/__mount/gencode-remote @@ -39,7 +39,7 @@ case "$state_should" in printf ' -o %s' "$(cat "$__object/parameter/options")" fi printf ' %s' "$(cat "$__object/parameter/device")" - printf " %s\\n" "$path" + printf ' %s\n' "$path" else # mount using existing fstab entry printf 'mount "%s"\n' "$path" diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index 592d4d0f..b72544c1 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -95,7 +95,7 @@ if [ -n "$curr_version" ]; then # PKG *is* installed if [ "$upgrade" = "true" ]; then execcmd "upg" "${cmd}" else - printf "Version %s is already installed and pkg-ng can't upgrade directly to version %s.\\nTo upgrade to the latest version, use the --upgrade flag.\\n" "$curr_version" "$version" >&2 + printf 'Version %s is already installed and pkg-ng cannot upgrade directly to version %s.\nTo upgrade to the latest version, use the --upgrade flag.\n' "$curr_version" "$version" >&2 exit 1 fi # PKG is supposed to be installed to the latest version From 334201b9452ef8bc6207e7cc8b118dc1ba0607e5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 10:20:08 +0200 Subject: [PATCH 172/178] Fix remaining shellcheck problems. --- cdist/conf/type/__daemontools/manifest | 1 + cdist/conf/type/__grafana_dashboard/manifest | 1 + cdist/conf/type/__jail/manifest | 2 +- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- cdist/conf/type/__mount/manifest | 2 +- cdist/conf/type/__postfix_master/manifest | 1 - cdist/conf/type/__process/gencode-remote | 3 +-- cdist/conf/type/__prometheus_alertmanager/manifest | 1 + cdist/conf/type/__prometheus_exporter/manifest | 1 + cdist/conf/type/__prometheus_server/manifest | 1 + cdist/conf/type/__ssh_authorized_keys/manifest | 2 +- 11 files changed, 10 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__daemontools/manifest b/cdist/conf/type/__daemontools/manifest index 656f4984..b04c7e07 100755 --- a/cdist/conf/type/__daemontools/manifest +++ b/cdist/conf/type/__daemontools/manifest @@ -9,6 +9,7 @@ __directory "$servicedir" --mode 700 os=$(cat "$__global/explorer/os") init=$(cat "$__global/explorer/init") +require="" case $os in freebsd) # TODO change to __start_on_boot once it supports freebsd diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 2e9bd115..9cd1465d 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -3,6 +3,7 @@ os=$(cat "$__global/explorer/os") os_version=$(cat "$__global/explorer/os_version") +require="" case $os in debian|devuan) case $os_version in diff --git a/cdist/conf/type/__jail/manifest b/cdist/conf/type/__jail/manifest index adf9bd99..fad6a3a1 100755 --- a/cdist/conf/type/__jail/manifest +++ b/cdist/conf/type/__jail/manifest @@ -37,7 +37,7 @@ jaildir="$(cat "$__object/parameter/jaildir")" __directory "${jaildir}" --parents -set -- "$@" "$__object_id" "--state" "$state" +set -- "$@" "$__object_id" cd "$__object/parameter" for property in *; do set -- "$@" "--$property" "$(cat "$property")" diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 56e3532c..d6892c9b 100755 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -91,7 +91,7 @@ if [ -f "${__object}/parameter/automatic-renewal" ]; then renew_hook_param="${__object}/parameter/renew-hook" renew_hook="" if [ -f "${renew_hook_param}" ]; then - while read hook; do + while read -r hook; do renew_hook="${renew_hook} --renew-hook \"${hook}\"" done < "${renew_hook_param}" fi diff --git a/cdist/conf/type/__mount/manifest b/cdist/conf/type/__mount/manifest index 73937899..999d806c 100755 --- a/cdist/conf/type/__mount/manifest +++ b/cdist/conf/type/__mount/manifest @@ -31,7 +31,7 @@ printf " %s" "$type" options="$(cat "$__object/parameter/options")" printf " %s" "$options" printf " %s" "$(cat "$__object/parameter/dump")" -printf " %s\n" "$(cat "$__object/parameter/pass")" +printf ' %s\n' "$(cat "$__object/parameter/pass")" ) | \ __block "$__object_name" \ --file "/etc/fstab" \ diff --git a/cdist/conf/type/__postfix_master/manifest b/cdist/conf/type/__postfix_master/manifest index 94b5044f..0960ea41 100755 --- a/cdist/conf/type/__postfix_master/manifest +++ b/cdist/conf/type/__postfix_master/manifest @@ -50,7 +50,6 @@ entry="$__object/files/entry" echo "# $(cat "$__object/parameter/comment")" fi printf "%s " "$service" - printf "%s " "$type" for parameter in type private unpriv chroot wakeup maxproc; do printf "%s " "$(cat "$__object/parameter/$parameter")" done diff --git a/cdist/conf/type/__process/gencode-remote b/cdist/conf/type/__process/gencode-remote index d14cf506..ec9691b9 100755 --- a/cdist/conf/type/__process/gencode-remote +++ b/cdist/conf/type/__process/gencode-remote @@ -52,8 +52,7 @@ case "$state_should" in if [ -f "$__object/parameter/stop" ]; then cat "$__object/parameter/stop" else - # TODO: what should be runs variable? - echo kill "${runs}" + echo kill "$(cat "$__object/parameter/runs")" fi echo "stopped" >> "$__messages_out" ;; diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest index 3f417526..64ef76af 100755 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -15,6 +15,7 @@ storage_path="$(cat "$__object/parameter/storage-path")" ##### INSTALL THE PACKAGE ################################################### require_pkg="" # what to require if I want to require "the package" +require="" if [ -f "$__object/parameter/install-from-backports" ]; then os=$(cat "$__global/explorer/os") os_version=$(cat "$__global/explorer/os_version") diff --git a/cdist/conf/type/__prometheus_exporter/manifest b/cdist/conf/type/__prometheus_exporter/manifest index 35e654e2..b9e14531 100644 --- a/cdist/conf/type/__prometheus_exporter/manifest +++ b/cdist/conf/type/__prometheus_exporter/manifest @@ -7,6 +7,7 @@ exporter="$(cat "$__object/parameter/exporter")" __user prometheus --system +require="" case $exporter in node) TEXTFILES=/service/node-exporter/textfiles # path for the textfiles collector diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest index 63b7b438..8685130f 100755 --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -18,6 +18,7 @@ rule_files="$(cat "$__object/parameter/rule-files")" ##### INSTALL THE PACKAGE ################################################### require_pkg="" # what to require if I want to require "the package" +require="" if [ -f "$__object/parameter/install-from-backports" ]; then os=$(cat "$__global/explorer/os") os_version=$(cat "$__global/explorer/os_version") diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest index 8cac3449..f6ff74c3 100755 --- a/cdist/conf/type/__ssh_authorized_keys/manifest +++ b/cdist/conf/type/__ssh_authorized_keys/manifest @@ -61,7 +61,7 @@ _cksum() { echo "$1" | cksum | cut -d' ' -f 1 } -while read key; do +while read -r key; do type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" object_id="$(_cksum "$file")-$(_cksum "$type_and_key")" set -- "$object_id" From cb826ad5708a3ceaadd6616098fbb9cd793ee652 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 18:47:38 +0200 Subject: [PATCH 173/178] ++changelog --- docs/changelog | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/docs/changelog b/docs/changelog index d7d31f52..1a3a7e2a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,21 +4,7 @@ Changelog next: * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) * Core: Add timestamp (optional) to log messages (Darko Poljak) - * Explorers, manifests, gencodes: Fix SC2166: and/or operators in test (shellcheck) (Jonas Weber, Thomas Eckert) - * Explorers and types: Fix SC2004: $/${} is unnecessary on arithmetic variables (shellcheck) (Darko Poljak) - * Explorers and types: Fix SC2005: Useless echo? Instead of echo $(cmd), just use cmd (shellcheck) (Darko Poljak) - * Explorers and types: Fix SC2002: Useless cat. Consider 'cmd < file | ..' or 'cmd file | ..' instead (shellcheck) (Darko Poljak) - * Explorers and types: Fix SC2039: In POSIX sh, something is undefined (shellcheck) (Darko Poljak) - * Explorers and types: Fix SC2045: Iterating over ls output is fragile. Use globs. (shellcheck) (Thomas Eckert) - * Explorers and types: Fix SC2148: Tips depend on target shell and yours is unknown. Add a shebang. (shellcheck) (Thomas Eckert) - * Explorers and types: Fix SC2069: To redirect stdout+stderr, 2>&1 must be last (or use '{ cmd > file; } 2>&1' to clarify). (shellcheck) (Jonas Weber) - * Explorers and types: Fix SC2034: foo appears unused. Verify it or export it. (shellcheck) (Jonas Weber) - * Explorers and types: Fix SC2215, SC2059, SC2155, SC2091. (shellcheck) (Jonas Weber) - * Explorers and types: Fix SC2086: Double quote to prevent globbing and word splitting. (shellcheck) (Jonas Weber) - * Explorers and types: Fix SC2164: Use cd ... || exit in case cd fails. (shellcheck) (Darko Poljak) - * Explorers and types: Disable SC2016 where it is intended: Expressions don't expand in single quotes, use double quotes for that. (shellcheck) (Darko Poljak) - * Explorers and types: Fix SC2044: For loops over find output are fragile. Use find -exec or a while read loop. (shellcheck) (Darko Poljak) - * Explorers and types: Fix SC1090,SC2021,SC2028,SC2030,SC2031,SC2039,SC2046,SC2116,SC2119,SC2120,SC2143,SC2145,SC2153,SC2162,SC2196,SC2230,SC2235. (shellcheck) (Darko Poljak) + * Explorers and types: Fix shellcheck found problems and encountered bugs (Jonas Weber, Thomas Eckert, Darko Poljak) * Build: Add shellcheck makefile target and check when doing release (Darko Poljak) 4.10.3: 2018-09-23 From ab8f87864e2be88a9529eb68f346f6139f7443c1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 19:09:14 +0200 Subject: [PATCH 174/178] Add shellcheck-ing for type explorers and auxiliary files. --- Makefile | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e9cacced..9f81cf06 100644 --- a/Makefile +++ b/Makefile @@ -262,6 +262,9 @@ pep8: shellcheck-global-explorers: @find cdist/conf/explorer -type f -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 +shellcheck-type-explorers: + @find cdist/conf/type -type f -path "*/explorer/*" -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 + shellcheck-manifests: @find cdist/conf/type -type f -name manifest -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 @@ -273,6 +276,11 @@ shellcheck-remote-gencodes: shellcheck-gencodes: shellcheck-local-gencodes shellcheck-remote-gencodes -shellcheck-types: shellcheck-manifests shellcheck-gencodes +shellcheck-types: shellcheck-type-explorers shellcheck-manifests shellcheck-gencodes shellcheck: shellcheck-global-explorers shellcheck-types + +shellcheck-type-files: + @find cdist/conf/type -type f -path "*/files/*" -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0 + +shellcheck-with-files: shellcheck shellcheck-type-files From 638ddd95d6b7f07c88baa44a8c6aa1022bf405e0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 19:41:44 +0200 Subject: [PATCH 175/178] Fix problems found by sellcheck in type explorers. --- cdist/conf/type/__apt_ppa/explorer/state | 3 ++- .../conf/type/__go_get/explorer/go-executable | 2 ++ .../type/__jail_freebsd10/explorer/status | 2 +- .../conf/type/__jail_freebsd9/explorer/status | 2 +- .../type/__postgres_database/explorer/state | 2 +- .../conf/type/__postgres_role/explorer/state | 2 +- .../type/__ssh_authorized_key/explorer/entry | 2 +- .../conf/type/__start_on_boot/explorer/state | 21 ++++++++++++++++--- .../type/__zypper_service/explorer/service_id | 2 +- 9 files changed, 28 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__apt_ppa/explorer/state b/cdist/conf/type/__apt_ppa/explorer/state index 2bb4f65a..d47e7d20 100755 --- a/cdist/conf/type/__apt_ppa/explorer/state +++ b/cdist/conf/type/__apt_ppa/explorer/state @@ -23,10 +23,11 @@ name="$__object_id" +# shellcheck disable=SC1091 . /etc/lsb-release repo_name="${name#ppa:}" -repo_file_name="$(echo "$repo_name" | sed -e "s|[/:]|-|" -e "s|\.|_|")-${DISTRIB_CODENAME}.list" +repo_file_name="$(echo "$repo_name" | sed -e 's|[/:]|-|' -e 's|\.|_|')-${DISTRIB_CODENAME}.list" [ -s "/etc/apt/sources.list.d/${repo_file_name}" ] \ && echo present || echo absent diff --git a/cdist/conf/type/__go_get/explorer/go-executable b/cdist/conf/type/__go_get/explorer/go-executable index bdce7559..87182282 100755 --- a/cdist/conf/type/__go_get/explorer/go-executable +++ b/cdist/conf/type/__go_get/explorer/go-executable @@ -1,4 +1,6 @@ #!/bin/sh +# shellcheck disable=SC1091 [ -f /etc/environment ] && . /etc/environment +# shellcheck disable=SC1091 [ -f /etc/profile ] && . /etc/profile go version 2>/dev/null || true diff --git a/cdist/conf/type/__jail_freebsd10/explorer/status b/cdist/conf/type/__jail_freebsd10/explorer/status index 1ceba212..c8039f21 100755 --- a/cdist/conf/type/__jail_freebsd10/explorer/status +++ b/cdist/conf/type/__jail_freebsd10/explorer/status @@ -39,7 +39,7 @@ fi # backslash-escaped $jaildir sjaildir="$(echo ${jaildir} | sed 's#/#\\/#g')" -jls_output="$(jls | grep "[ ]${sjaildir}\/${name}\$")" || true +jls_output="$(jls | grep "[ ]${sjaildir}\\/${name}\$")" || true if [ -n "${jls_output}" ]; then echo "STARTED" diff --git a/cdist/conf/type/__jail_freebsd9/explorer/status b/cdist/conf/type/__jail_freebsd9/explorer/status index 1ceba212..c8039f21 100755 --- a/cdist/conf/type/__jail_freebsd9/explorer/status +++ b/cdist/conf/type/__jail_freebsd9/explorer/status @@ -39,7 +39,7 @@ fi # backslash-escaped $jaildir sjaildir="$(echo ${jaildir} | sed 's#/#\\/#g')" -jls_output="$(jls | grep "[ ]${sjaildir}\/${name}\$")" || true +jls_output="$(jls | grep "[ ]${sjaildir}\\/${name}\$")" || true if [ -n "${jls_output}" ]; then echo "STARTED" diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index dc9659e2..54eb768d 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -20,7 +20,7 @@ name="$__object_id" -if su - postgres -c "echo '\q' | psql '$name'" 2>/dev/null; then +if su - postgres -c "echo '\\q' | psql '$name'" 2>/dev/null; then echo "present" else echo "absent" diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 8c102df9..40f64cef 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -20,7 +20,7 @@ name="$__object_id" -if su - postgres -c "psql -c '\du' | grep -q '^ *$name *|'"; then +if su - postgres -c "psql -c '\\du' | grep -q '^ *$name *|'"; then echo "present" else echo "absent" diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index b528b26e..78efbb48 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -27,5 +27,5 @@ then file="$(cat "$__object/parameter/file")" # get any entries that match the type and key - grep ".*$type_and_key\([ \n]\|$\)" "$file" || true + grep ".*$type_and_key\\([ \\n]\\|$\\)" "$file" || true fi diff --git a/cdist/conf/type/__start_on_boot/explorer/state b/cdist/conf/type/__start_on_boot/explorer/state index d8f9b7ba..cef9013e 100644 --- a/cdist/conf/type/__start_on_boot/explorer/state +++ b/cdist/conf/type/__start_on_boot/explorer/state @@ -38,12 +38,27 @@ if [ "$init" = 'systemd' ]; then else case "$os" in debian|openwrt|devuan) - state="present" - [ -f "/etc/rc$runlevel.d/S"??"$name" ] || state="absent" + state="absent" + for file in "/etc/rc$runlevel.d/S"??"$name" + do + if [ -f "$file" ] + then + state="present" + break + fi + done ;; ubuntu) state="absent" - [ -f "/etc/rc$runlevel.d/S"??"$name" ] && state="present" + for file in "/etc/rc$runlevel.d/S"??"$name" + do + if [ -f "$file" ] + then + state="present" + break + fi + done + [ -f "/etc/init/${name}.conf" ] && state="present" ;; diff --git a/cdist/conf/type/__zypper_service/explorer/service_id b/cdist/conf/type/__zypper_service/explorer/service_id index 91858d84..7161f804 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_id +++ b/cdist/conf/type/__zypper_service/explorer/service_id @@ -27,4 +27,4 @@ else fi # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper ls -u -E | grep -E "\<$uri\>" | cut -d'|' -f 1 ) -zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\<$uri\>" | cut -d'|' -f 1 +zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\\<$uri\\>" | cut -d'|' -f 1 From ec186673bf4021d67221d011921ce7165e9473f0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 7 Oct 2018 19:46:34 +0200 Subject: [PATCH 176/178] Add TODO markers for SC2012. --- cdist/conf/type/__link/explorer/state | 3 +++ cdist/conf/type/__link/explorer/type | 1 + 2 files changed, 4 insertions(+) diff --git a/cdist/conf/type/__link/explorer/state b/cdist/conf/type/__link/explorer/state index 587835a2..edccf08d 100755 --- a/cdist/conf/type/__link/explorer/state +++ b/cdist/conf/type/__link/explorer/state @@ -33,6 +33,7 @@ destination_dir="${destination%/*}" case "$type" in symbolic) cd "$destination_dir" || exit 1 + # TODO SC2012: use readlink or something? source_is=$(ls -l "$destination" | sed 's/.*-> //g') if [ -h "$destination" ]; then # ignore trailing slashes for comparison @@ -52,7 +53,9 @@ case "$type" in echo sourcemissing exit 0 fi + # TODO SC2012: use stat? destination_inode=$(ls -i "$destination" | awk '{print $1}') + # TODO SC2012: use stat? source_inode=$(ls -i "$source" | awk '{print $1}') if [ "$destination_inode" -eq "$source_inode" ]; then echo present diff --git a/cdist/conf/type/__link/explorer/type b/cdist/conf/type/__link/explorer/type index 48a79bf0..aa1aff2c 100755 --- a/cdist/conf/type/__link/explorer/type +++ b/cdist/conf/type/__link/explorer/type @@ -31,6 +31,7 @@ elif [ -f "$destination" ]; then type="$(cat "$__object/parameter/type")" case "$type" in hard) + # TODO SC2012: use stat? link_count=$(ls -l "$destination" | awk '{ print $2 }') if [ "$link_count" -gt 1 ]; then echo hardlink From 0458f66c96b1061b81f759cfa47f8fe20f2d66ae Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Oct 2018 15:35:29 +0200 Subject: [PATCH 177/178] Resolve SC2012. --- cdist/conf/type/__link/explorer/state | 11 +++++---- cdist/conf/type/__link/explorer/type | 34 ++++++++++++++------------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/cdist/conf/type/__link/explorer/state b/cdist/conf/type/__link/explorer/state index edccf08d..a601d53b 100755 --- a/cdist/conf/type/__link/explorer/state +++ b/cdist/conf/type/__link/explorer/state @@ -33,8 +33,7 @@ destination_dir="${destination%/*}" case "$type" in symbolic) cd "$destination_dir" || exit 1 - # TODO SC2012: use readlink or something? - source_is=$(ls -l "$destination" | sed 's/.*-> //g') + source_is=$(readlink "$destination") if [ -h "$destination" ]; then # ignore trailing slashes for comparison if [ "${source_is%/}" = "${source%/}" ]; then @@ -53,9 +52,13 @@ case "$type" in echo sourcemissing exit 0 fi - # TODO SC2012: use stat? + # Currently not worth the effor to change it, stat is not defined by POSIX + # and different OSes has different implementations for it. + # shellcheck disable=SC2012 destination_inode=$(ls -i "$destination" | awk '{print $1}') - # TODO SC2012: use stat? + # Currently not worth the effor to change it, stat is not defined by POSIX + # and different OSes has different implementations for it. + # shellcheck disable=SC2012 source_inode=$(ls -i "$source" | awk '{print $1}') if [ "$destination_inode" -eq "$source_inode" ]; then echo present diff --git a/cdist/conf/type/__link/explorer/type b/cdist/conf/type/__link/explorer/type index aa1aff2c..796f1ac6 100755 --- a/cdist/conf/type/__link/explorer/type +++ b/cdist/conf/type/__link/explorer/type @@ -24,24 +24,26 @@ destination="/$__object_id" if [ ! -e "$destination" ]; then - echo none + echo none elif [ -h "$destination" ]; then - echo symlink + echo symlink elif [ -f "$destination" ]; then - type="$(cat "$__object/parameter/type")" - case "$type" in - hard) - # TODO SC2012: use stat? - link_count=$(ls -l "$destination" | awk '{ print $2 }') - if [ "$link_count" -gt 1 ]; then - echo hardlink - exit 0 - fi - ;; - esac - echo file + type="$(cat "$__object/parameter/type")" + case "$type" in + hard) + # Currently not worth the effor to change it, stat is not defined by POSIX + # and different OSes has different implementations for it. + # shellcheck disable=SC2012 + ink_count=$(ls -l "$destination" | awk '{ print $2 }') + if [ "$link_count" -gt 1 ]; then + echo hardlink + exit 0 + fi + ;; + esac + echo file elif [ -d "$destination" ]; then - echo directory + echo directory else - echo unknown + echo unknown fi From 69790ec138a9ef63c179b5d2ca3777c758ed7347 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Oct 2018 15:43:42 +0200 Subject: [PATCH 178/178] Fix typing error. --- cdist/conf/type/__link/explorer/type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__link/explorer/type b/cdist/conf/type/__link/explorer/type index 796f1ac6..b322bf42 100755 --- a/cdist/conf/type/__link/explorer/type +++ b/cdist/conf/type/__link/explorer/type @@ -34,7 +34,7 @@ elif [ -f "$destination" ]; then # Currently not worth the effor to change it, stat is not defined by POSIX # and different OSes has different implementations for it. # shellcheck disable=SC2012 - ink_count=$(ls -l "$destination" | awk '{ print $2 }') + link_count=$(ls -l "$destination" | awk '{ print $2 }') if [ "$link_count" -gt 1 ]; then echo hardlink exit 0