From 553c11ca95ea2f8d6ad1f3a8e653784754bcb9e6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:22:31 +0100 Subject: [PATCH 001/138] gitlab CI runner should have necessary tools --- .gitlab-ci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 44de6b0b..1cc17995 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,26 +3,16 @@ stages: unit_tests: stage: test - before_script: - - 'apk update' - - 'apk add python3' script: - ./bin/build-helper version - ./bin/build-helper test pycodestyle: stage: test - before_script: - - 'apk update' - - 'apk add python3 py3-pycodestyle' - - 'ln -f -s /usr/bin/pycodestyle-3 /usr/bin/pycodestyle' script: - ./bin/build-helper pycodestyle shellcheck: stage: test - before_script: - - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz' - - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/' script: - ./bin/build-helper shellcheck From 75b2f521d932c1098f9f1502cbc758f8ad434faa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:25:48 +0100 Subject: [PATCH 002/138] debug runner :) --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1cc17995..6a429b50 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ unit_tests: pycodestyle: stage: test script: + - ls -l $(command -v pycodestyle) - ./bin/build-helper pycodestyle shellcheck: From 936019b699db7856bf175dec7a96f09ee88f531d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:31:20 +0100 Subject: [PATCH 003/138] debug --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a429b50..346f0d09 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,6 +10,7 @@ unit_tests: pycodestyle: stage: test script: + - $(command -v pycodestyle) - ls -l $(command -v pycodestyle) - ./bin/build-helper pycodestyle From f259c93796071af47419752648418b3def75701c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:33:41 +0100 Subject: [PATCH 004/138] debug --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 346f0d09..c2ea902e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,6 +11,7 @@ pycodestyle: stage: test script: - $(command -v pycodestyle) + - $(command -v pycodestyle3) - ls -l $(command -v pycodestyle) - ./bin/build-helper pycodestyle From 399828545f8d2920ecc544b9f4ed3b074f6d861b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:35:18 +0100 Subject: [PATCH 005/138] debug --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c2ea902e..caa939e5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,8 +10,8 @@ unit_tests: pycodestyle: stage: test script: - - $(command -v pycodestyle) - $(command -v pycodestyle3) + - $(command -v pycodestyle) - ls -l $(command -v pycodestyle) - ./bin/build-helper pycodestyle From 20ccb3ec06e24c69127b8026f2b525559ef35fd4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 11:33:53 +0100 Subject: [PATCH 006/138] debug --- .gitlab-ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index caa939e5..f6babf74 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,9 +10,7 @@ unit_tests: pycodestyle: stage: test script: - - $(command -v pycodestyle3) - - $(command -v pycodestyle) - - ls -l $(command -v pycodestyle) + - pep8=$(command -v pycodestyle-3) && ln -s -f "${pep8}" /usr/bin/pycodestyle - ./bin/build-helper pycodestyle shellcheck: From 654637c9dd28fff8de382df016d8d1cc41e33c58 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 12:31:46 +0100 Subject: [PATCH 007/138] debug --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f6babf74..1cc17995 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,7 +10,6 @@ unit_tests: pycodestyle: stage: test script: - - pep8=$(command -v pycodestyle-3) && ln -s -f "${pep8}" /usr/bin/pycodestyle - ./bin/build-helper pycodestyle shellcheck: From 8c0ee28b9940b765e48e839008b8f3765567da8e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 12:35:43 +0100 Subject: [PATCH 008/138] gitlab CI runner should have necessary tools --- .gitlab-ci.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 44de6b0b..1cc17995 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,26 +3,16 @@ stages: unit_tests: stage: test - before_script: - - 'apk update' - - 'apk add python3' script: - ./bin/build-helper version - ./bin/build-helper test pycodestyle: stage: test - before_script: - - 'apk update' - - 'apk add python3 py3-pycodestyle' - - 'ln -f -s /usr/bin/pycodestyle-3 /usr/bin/pycodestyle' script: - ./bin/build-helper pycodestyle shellcheck: stage: test - before_script: - - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz' - - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/' script: - ./bin/build-helper shellcheck From d25c72e678f8d4fb311a9800a3097e1bf6e9bf16 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Tue, 3 Dec 2019 10:13:46 +0100 Subject: [PATCH 009/138] [PACKAGE-UPDATE-INDEX]: Bugfix of the alpine part of the __package_update_index type --- cdist/conf/type/__package_update_index/gencode-remote | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 9b2ecba2..6c51cbed 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -47,9 +47,9 @@ case "$type" in echo "pacman --noprogressbar --sync --refresh" echo "pacman package database synced (age was: $currage)" >> "$__messages_out" ;; - alpine) + apk) echo "apk update" - echo "apk package database updated." + echo "apk package database updated." >>"$__messages_out" ;; *) echo "Don't know how to manage packages for type: $type" >&2 From 320f962e1df7f27909e6157e8dfc6033706c8ddd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 4 Dec 2019 22:27:58 +0100 Subject: [PATCH 010/138] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 09e30e79..8bdee928 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __package_update_index: Fix Alpine part (Dominique Roux) + 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) From 4f21bf534dd91c7395845beb08715ad6c84bafef Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 5 Dec 2019 23:14:27 +0100 Subject: [PATCH 011/138] [docs] PreOS: English nitpicking. --- cdist/argparse.py | 2 +- docs/src/cdist-preos.rst | 70 ++++++++++++++++++++-------------------- 2 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index ca69cdae..ed94d266 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -424,7 +424,7 @@ def get_parsers(): parser['inventory'].set_defaults( func=cdist.inventory.Inventory.commandline) - # PreOs + # PreOS parser['preos'] = parser['sub'].add_parser('preos', add_help=False) # Shell diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst index e85af2de..2f102fef 100644 --- a/docs/src/cdist-preos.rst +++ b/docs/src/cdist-preos.rst @@ -4,9 +4,9 @@ PreOS Description ----------- With cdist you can install and configure new machines. You can use cdist to -create PreOS, minimal OS which purpose is to boot new machine. -After PreOS is booted machine is ready for installing desired OS and -then it is ready for configuration. +create PreOS, minimal OS whose purpose is to boot a new machine. +After PreOS is booted, the machine is ready for installing the desired OS and +afterwards it is ready for configuration. PreOS creation -------------- @@ -15,62 +15,62 @@ Currently supported PreOS-es include: * debian * ubuntu -* devuan. +* devuan -PreOS is created using cdist preos command. preos command has subcommands that -create the desired PreOS. +PreOS is created using the ``cdist preos`` command. +This command has subcommands that determine the desired PreOS. -For example, to create ubuntu PreOS: +For example, to create an ubuntu PreOS: .. code-block:: sh $ cdist preos ubuntu /preos/preos-ubuntu -B -C \ -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu -For more info about available options see cdist manual page. +For more info about the available options see the cdist manual page. -This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it -will be configured (``-C``) using default built-in initial manifest and with -specified ssh authorized key (``-k``). -After bootstrapping and configuration PXE -boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``. +This will bootstrap (``-B``) ubuntu PreOS in the ``/preos/preos-ubuntu`` +directory, it will be configured (``-C``) using default the built-in initial +manifest and with specified ssh authorized key (``-k``). +After bootstrapping and configuration, the PXE boot directory will be +created (``-p``) in ``/preos/pxe-ubuntu``. -After PreOS is created new machines can be booted using created PXE (after -proper dhcp, tftp setting). +After PreOS is created, new machines can be booted using the created PXE +(after proper dhcp and tftp settings). -Since PreOS is configured with ssh authorized key it can be accessed throguh +Since PreOS is configured with ssh authorized key it can be accessed through ssh, i.e. it can be further installed and configured with cdist. -Implementing new PreOS sub-command ----------------------------------- +Implementing a new PreOS sub-command +------------------------------------ preos command is implemented as a plugin system. This plugin system scans for -preos subcommands in ``cdist/preos/`` distribution directory and also in +preos subcommands in the ``cdist/preos/`` distribution directory and also in ``~/.cdist/preos/`` directory if it exists. preos subcommand is a module or a class that satisfies the following: -* it has attribute ``_cdist_preos`` set to ``True`` -* it has function/method ``commandline``. +* it has the attribute ``_cdist_preos`` set to ``True`` +* it defines a function/method ``commandline``. -For a module based preos subcommand ``commandline`` function accepts a module -object as its first argument and the list of command line +For a module-based preos subcommand, the ``commandline`` function accepts a +module object as its first argument and the list of command line arguments (``sys.argv[2:]``). -For a class preos subcommand ``commandline`` method should be staticmethod and -it accepts a class object as its first argument and the list of command line -arguments(``sys.argv[2:]``). +For a class-based preos subcommand ``commandline`` method should be +static-method and must accept a class as its first argument and the +list of command line arguments (``sys.argv[2:]``). If preos scanning finds a module/class that has ``_cdist_preos`` set -to ``True`` and it has function/method ``commandline`` then this module/class is +to ``True`` and a function/method ``commandline`` then this module/class is registered to preos subcommands. The name of the command is set to ``_preos_name`` -attribute if it exists, otherwise it is set to the module/class name, lowercase. -When registered preos subcommand is specified as preos command then ``commandline`` -will be called with first argument set to module/class object and second argument -set to ``sys.argv[2:]``. +attribute if defined in the module/class, defaulting to the module/class name in lowercase. +When a registered preos subcommand is specified, ``commandline`` +will be called with the first argument set to module/class and the second +argument set to ``sys.argv[2:]``. -Example writing new dummy preos sub-command -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Module based preos: +Example of writing new dummy preos sub-command +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Module-based preos: ^^^^^^^^^^^^^^^^^^^ #. Create directory ``~/.cdist/preos/`` if it does not exist @@ -134,4 +134,4 @@ When you try to run this new preos you will get: FreeBSD dummy preos: [] In the ``commandline`` function/method you have all the freedom to actually create -PreOS. +a PreOS. From f22349ce8a6571601478a76bfe3b68bb574154ba Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 6 Dec 2019 08:56:57 +0100 Subject: [PATCH 012/138] Fix documentation for install types Fixes gitlab #790. --- cdist/conf/type/__install_directory/man.rst | 102 +++++++++++++++++++- cdist/conf/type/__install_file/man.rst | 14 ++- docs/changelog | 1 + 3 files changed, 115 insertions(+), 2 deletions(-) mode change 120000 => 100644 cdist/conf/type/__install_directory/man.rst diff --git a/cdist/conf/type/__install_directory/man.rst b/cdist/conf/type/__install_directory/man.rst deleted file mode 120000 index 1ad7fa84..00000000 --- a/cdist/conf/type/__install_directory/man.rst +++ /dev/null @@ -1 +0,0 @@ -../__directory/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_directory/man.rst b/cdist/conf/type/__install_directory/man.rst new file mode 100644 index 00000000..c402cbad --- /dev/null +++ b/cdist/conf/type/__install_directory/man.rst @@ -0,0 +1,101 @@ +cdist-type__install_directory(7) +================================ + +NAME +---- +cdist-type__install_directory - Manage a directory with install command + + +DESCRIPTION +----------- +This cdist type allows you to create or remove directories on the target. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +state + 'present' or 'absent', defaults to 'present' + +group + Group to chgrp to. + +mode + Unix permissions, suitable for chmod. + +owner + User to chown to. + + +BOOLEAN PARAMETERS +------------------ +parents + Whether to create parents as well (mkdir -p behaviour). + Warning: all intermediate directory permissions default + to whatever mkdir -p does. + + Usually this means root:root, 0700. + +recursive + If supplied the chgrp and chown call will run recursively. + This does *not* influence the behaviour of chmod. + +MESSAGES +-------- +chgrp + Changed group membership +chown + Changed owner +chmod + Changed mode +create + Empty directory was created +remove + Directory exists, but state is absent, directory will be removed by generated code. +remove non directory + Something other than a directory with the same name exists and was removed prior to create. + + +EXAMPLES +-------- + +.. code-block:: sh + + # A silly example + __install_directory /tmp/foobar + + # Remove a directory + __install_directory /tmp/foobar --state absent + + # Ensure /etc exists correctly + __install_directory /etc --owner root --group root --mode 0755 + + # Create nfs service directory, including parents + __install_directory /home/services/nfs --parents + + # Change permissions recursively + __install_directory /home/services --recursive --owner root --group root + + # Setup a temp directory + __install_directory /local --mode 1777 + + # Take it all + __install_directory /home/services/kvm --recursive --parents \ + --owner root --group root --mode 0755 --state present + + +AUTHORS +------- +Nico Schottelius + + +COPYING +------- +Copyright \(C) 2011 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_file/man.rst b/cdist/conf/type/__install_file/man.rst index c5409167..977ed77c 100644 --- a/cdist/conf/type/__install_file/man.rst +++ b/cdist/conf/type/__install_file/man.rst @@ -23,6 +23,10 @@ symlink directory replace it with the source file +One exception is that when state is pre-exists, an error is raised if +the file would have been created otherwise (e.g. it is not present or +not a regular file). + In any case, make sure that the file attributes are as specified. @@ -33,7 +37,7 @@ None. OPTIONAL PARAMETERS ------------------- state - 'present', 'absent' or 'exists', defaults to 'present' where: + 'present', 'absent', 'exists' or 'pre-exists', defaults to 'present' where: present the file is exactly the one from source @@ -41,6 +45,9 @@ state the file does not exist exists the file from source but only if it doesn't already exist + pre-exists + check that the file exists and is a regular file, but do not + create or modify it group Group to chgrp to. @@ -56,6 +63,9 @@ source If not supplied, an empty file or directory will be created. If source is '-' (dash), take what was written to stdin as the file content. +onchange + The code to run if file is modified. + MESSAGES -------- chgrp @@ -93,6 +103,8 @@ EXAMPLES __install_file /home/frodo/.bashrc --source "/etc/skel/.bashrc" \ --state exists \ --owner frodo --mode 0600 + # Check that the file is present, show an error when it is not + __install_file /etc/somefile --state pre-exists # Take file content from stdin __install_file /tmp/whatever --owner root --group root --mode 644 --source - << DONE Here goes the content for /tmp/whatever diff --git a/docs/changelog b/docs/changelog index 8bdee928..ed993609 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __package_update_index: Fix Alpine part (Dominique Roux) + * Documentation: Fix man pages for install types (Darko Poljak) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) From 90e85d81285a1ed60085b4df0a1618e4aeaa8c6f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 6 Dec 2019 09:37:21 +0100 Subject: [PATCH 013/138] Embed config skeleton instead of rewriting it Resolves #796. --- .gitignore | 1 + Makefile | 8 +++- docs/changelog | 1 + docs/src/cdist-configuration.rst | 70 ++------------------------------ 4 files changed, 13 insertions(+), 67 deletions(-) diff --git a/.gitignore b/.gitignore index ed8b453a..fb831051 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ docs/src/man1/*.1 docs/src/man7/*.7 docs/src/man7/cdist-type__*.rst docs/src/cdist-reference.rst +docs/src/cdist.cfg.skeleton # Ignore cdist cache for version control /cache/ diff --git a/Makefile b/Makefile index fa3327d1..e6a2a28e 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,11 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh $(DOCSREF): $(DOCSREFSH) $(DOCSREFSH) +DOCSCFGSKEL=./configuration/cdist.cfg.skeleton + +configskel: $(DOCSCFGSKEL) + cp -f "$(DOCSCFGSKEL)" "$(DOCS_SRC_DIR)/" + version: @[ -f "cdist/version.py" ] || { \ printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \ @@ -72,7 +77,7 @@ version: man: version $(MANTYPES) $(DOCSREF) $(SPHINXM) -html: version $(MANTYPES) $(DOCSREF) +html: version configskel $(MANTYPES) $(DOCSREF) $(SPHINXH) docs: man html @@ -114,6 +119,7 @@ speeches: $(SPEECHES) # clean: docs-clean rm -f $(DOCS_SRC_DIR)/cdist-reference.rst + rm -f $(DOCS_SRC_DIR)/cdist.cfg.skeleton find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \ | xargs rm -f diff --git a/docs/changelog b/docs/changelog index ed993609..ca33d7bd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __package_update_index: Fix Alpine part (Dominique Roux) * Documentation: Fix man pages for install types (Darko Poljak) + * Documentation: Embed config skeleton instead of rewriting it (Darko Poljak) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst index 4c9b4d33..706ed761 100644 --- a/docs/src/cdist-configuration.rst +++ b/docs/src/cdist-configuration.rst @@ -31,73 +31,11 @@ can be used. Config file format ------------------ + cdist configuration file is in the INI file format. Currently it supports only [GLOBAL] section. -The possible keywords and their meanings are as follows: -:strong:`archiving` - Use specified archiving. Valid values include: - 'none', 'tar', 'tgz', 'tbz2' and 'txz'. +Here you can find configuration file skeleton: -:strong:`beta` - Enable beta functionality. It recognizes boolean values from - 'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'. - -:strong:`cache_path_pattern` - Specify cache path pattern. - -:strong:`conf_dir` - List of configuration directories separated with the character conventionally - used by the operating system to separate search path components (as in PATH), - such as ':' for POSIX or ';' for Windows. - If also specified at command line then values from command line are - appended to this value. - -:strong:`init_manifest` - Specify default initial manifest. - -:strong:`inventory_dir` - Specify inventory directory. - -:strong:`jobs` - Specify number of jobs for parallel processing. If -1 then the default, - number of CPU's in the system is used. If 0 then parallel processing in - jobs is disabled. If set to positive number then specified maximum - number of processes will be used. - -:strong:`local_shell` - Shell command used for local execution. - -:strong:`out_path` - Directory to save cdist output in. - -:strong:`parallel` - Process hosts in parallel. If -1 then the default, number of CPU's in - the system is used. If 0 then parallel processing of hosts is disabled. - If set to positive number then specified maximum number of processes - will be used. - -:strong:`remote_copy` - Command to use for remote copy (should behave like scp). - -:strong:`remote_exec` - Command to use for remote execution (should behave like ssh). - -:strong:`remote_out_path` - Directory to save cdist output in on the target host. - -:strong:`remote_shell` - Shell command at remote host used for remote execution. - -:strong:`save_output_streams` - Enable/disable saving output streams (enabled by default). - It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false' - and '1'/'0'. - -:strong:`timestamp` - Timestamp log messages with the current local date and time - in the format: YYYYMMDDHHMMSS.us. - -:strong:`verbosity` - Set verbosity level. Valid values are: - 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. +.. literalinclude:: cdist.cfg.skeleton + :language: ini From 71b6646b38799f7316d791a4d9b511098d879ab0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 6 Dec 2019 10:00:32 +0100 Subject: [PATCH 014/138] Rm cdist-type prefix and man page ref from docs E.g. 16.10. __block 16.11. __ccollect_source 16.12. __cdist 16.13. __cdistmarker 16.14. __check_messages insted of 16.10. cdist-type__block(7) 16.11. cdist-type__ccollect_source(7) 16.12. cdist-type__cdist(7) 16.13. cdist-type__cdistmarker(7) 16.14. cdist-type__check_messages(7) --- .gitignore | 1 + Makefile | 10 ++++++- docs/src/cdist-types.rst | 8 ------ docs/src/cdist-types.rst.sh | 55 +++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 9 deletions(-) delete mode 100644 docs/src/cdist-types.rst create mode 100755 docs/src/cdist-types.rst.sh diff --git a/.gitignore b/.gitignore index fb831051..85a8ccc7 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ docs/src/man1/*.1 docs/src/man7/*.7 docs/src/man7/cdist-type__*.rst docs/src/cdist-reference.rst +docs/src/cdist-types.rst docs/src/cdist.cfg.skeleton # Ignore cdist cache for version control diff --git a/Makefile b/Makefile index e6a2a28e..f89ac1e7 100644 --- a/Makefile +++ b/Makefile @@ -63,6 +63,13 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh $(DOCSREF): $(DOCSREFSH) $(DOCSREFSH) +# Html types list with references +DOCSTYPESREF=$(MAN7DSTDIR)/cdist-types.rst +DOCSTYPESREFSH=$(DOCS_SRC_DIR)/cdist-types.rst.sh + +$(DOCSTYPESREF): $(DOCSTYPESREFSH) + $(DOCSTYPESREFSH) + DOCSCFGSKEL=./configuration/cdist.cfg.skeleton configskel: $(DOCSCFGSKEL) @@ -77,7 +84,7 @@ version: man: version $(MANTYPES) $(DOCSREF) $(SPHINXM) -html: version configskel $(MANTYPES) $(DOCSREF) +html: version configskel $(MANTYPES) $(DOCSREF) $(DOCSTYPESREF) $(SPHINXH) docs: man html @@ -119,6 +126,7 @@ speeches: $(SPEECHES) # clean: docs-clean rm -f $(DOCS_SRC_DIR)/cdist-reference.rst + rm -f $(DOCS_SRC_DIR)/cdist-types.rst rm -f $(DOCS_SRC_DIR)/cdist.cfg.skeleton find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \ diff --git a/docs/src/cdist-types.rst b/docs/src/cdist-types.rst deleted file mode 100644 index d5104667..00000000 --- a/docs/src/cdist-types.rst +++ /dev/null @@ -1,8 +0,0 @@ -cdist types -=========== - -.. toctree:: - :titlesonly: - :glob: - - man7/* diff --git a/docs/src/cdist-types.rst.sh b/docs/src/cdist-types.rst.sh new file mode 100755 index 00000000..2eb66009 --- /dev/null +++ b/docs/src/cdist-types.rst.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# +# 2019 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Generate cdist-types.rst that lists available types. +# + +__cdist_pwd="$(pwd -P)" +__cdist_mydir="${0%/*}"; +__cdist_abs_mydir="$(cd "$__cdist_mydir" && pwd -P)" +__cdist_myname=${0##*/}; +__cdist_abs_myname="$__cdist_abs_mydir/$__cdist_myname" + +filename="${__cdist_myname%.sh}" +dest="$__cdist_abs_mydir/$filename" + +cd "$__cdist_abs_mydir" + +exec > "$dest" +cat << eof +cdist types +=========== + +.. toctree:: + :titlesonly: + +eof + +# If there is no such file then ls prints error to stderr, +# so redirect stderr to /dev/null. +for type in $(ls man7/cdist-type__*.rst 2>/dev/null | LC_ALL=C sort); do + no_dir="${type#man7/}"; + no_type="${no_dir#cdist-type}"; + name="${no_type%.rst}"; + manref="${no_dir%.rst}" + man="${manref}(7)" + + echo " $name" "" +done From f2cbc01e3f885c41962a1e474922127c49099203 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 6 Dec 2019 10:04:06 +0100 Subject: [PATCH 015/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ca33d7bd..bf0398a9 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __package_update_index: Fix Alpine part (Dominique Roux) * Documentation: Fix man pages for install types (Darko Poljak) * Documentation: Embed config skeleton instead of rewriting it (Darko Poljak) + * Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) From 839e7a408e199cd5a0884c25e283c9bebe13e26d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 6 Dec 2019 19:46:17 +0100 Subject: [PATCH 016/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bf0398a9..f4e38578 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Documentation: Fix man pages for install types (Darko Poljak) * Documentation: Embed config skeleton instead of rewriting it (Darko Poljak) * Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak) + * Documentation: PreOS english nitpicking (Evil Ham) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) From d5ac9ea348aa0ed640941232fbc2c783d95b8750 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 8 Dec 2019 16:34:38 +0100 Subject: [PATCH 017/138] Docs: install from source, verify signature Add chapter for installing from source with signature verification. Resolves #795. --- docs/changelog | 1 + docs/src/cdist-install.rst | 27 +++++++++++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/docs/changelog b/docs/changelog index f4e38578..33db10da 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Documentation: Embed config skeleton instead of rewriting it (Darko Poljak) * Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak) * Documentation: PreOS english nitpicking (Evil Ham) + * Documentation: Add installing from source with signature verification (Darko Poljak) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index a9b7d6b5..880b9f8e 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -23,8 +23,6 @@ Target Hosts Install cdist ------------- -You can install cdist either from git or as a python package. - From git ~~~~~~~~ @@ -42,6 +40,7 @@ To install cdist, execute the following commands: From version 4.2.0 cdist tags and releases are signed. You can get GPG public key used for signing `here <_static/pgp-key-EFD2AE4EC36B6901.asc>`_. +It is assumed that you are familiar with *git* ways of signing and verification. You can also get cdist from `github mirror `_. @@ -157,3 +156,27 @@ Cdist is available as a python package at .. code-block:: sh pip install cdist + +Installing from source with signature verification +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +If you want to install cdist from signed source and verify it, first you need to +download cdist archive and its detached signature. + +Get both, *cdist-x.y.z.tar.gz* and *cdist-x.y.z.tar.gz.asc* from release +notes of the desired tag *x.y.z* at +`cdist git repository `_. + +Get GPG public key used for signing `here <_static/pgp-key-EFD2AE4EC36B6901.asc>`_ +and import it into GPG. + +Now cdist source archive can be verified using `gpg`, e.g. to verify `cdist-6.2.0`: + +.. code-block:: sh + + $ gpg --verify cdist-6.2.0.tar.gz.asc cdist-6.2.0.targ.gz + gpg: Signature made Sat Nov 30 23:14:19 2019 CET + gpg: using RSA key 69767822F3ECC3C349C1EFFFEFD2AE4EC36B6901 + gpg: Good signature from "ungleich GmbH (ungleich FOSS) " [ultimate] + +Further steps are the same as for `installing from git `_. From 4735df1bed2b23db8c16f568280ef1c5a8ed4c03 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 6 Dec 2019 19:40:05 +0100 Subject: [PATCH 018/138] Add plugins-dir preos option --- cdist/argparse.py | 2 +- cdist/preos.py | 72 +++++++++++++++++++++++++++++----------- docs/src/cdist-preos.rst | 30 ++++++----------- docs/src/man1/cdist.rst | 52 +++++++++++++++++------------ 4 files changed, 95 insertions(+), 61 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index ed94d266..7dc683f3 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -103,7 +103,7 @@ def get_parsers(): name="log level"), help=('Set the specified verbosity level. ' 'The levels, in order from the lowest to the highest, are: ' - 'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) ' + 'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3), ' 'TRACE (4 or higher). If used along with -v then -v ' 'increases last set value and -l overwrites last set ' 'value.'), diff --git a/cdist/preos.py b/cdist/preos.py index 46b45554..378071db 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -5,6 +5,8 @@ import inspect import argparse import cdist import logging +import re +import cdist.argparse _PREOS_CALL = "commandline" @@ -12,15 +14,24 @@ _PREOS_NAME = "_preos_name" _PREOS_MARKER = "_cdist_preos" _PLUGINS_DIR = "preos" _PLUGINS_PATH = [os.path.join(os.path.dirname(__file__), _PLUGINS_DIR), ] +log = logging.getLogger("PreOS") + + +def extend_plugins_path(dirs): + for dir in dirs: + preos_dir = os.path.expanduser(os.path.join(dir, "preos")) + if os.path.isdir(preos_dir): + _PLUGINS_PATH.append(preos_dir) + + cdist_home = cdist.home_dir() if cdist_home: - cdist_home_preos = os.path.join(cdist_home, "preos") - if os.path.isdir(cdist_home_preos): - _PLUGINS_PATH.append(cdist_home_preos) -sys.path.extend(_PLUGINS_PATH) - - -log = logging.getLogger("PreOS") + extend_plugins_path((cdist_home, )) +x = 'CDIST_PATH' +if x in os.environ: + vals = re.split(r'(? Date: Mon, 9 Dec 2019 09:57:54 +0100 Subject: [PATCH 019/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 33db10da..04ffac62 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Documentation: Remove cdist-type prefix and man page reference from type list in html (Darko Poljak) * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) + * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) From bee95efa4317eacbde98237f7952d020cdfb5e61 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 9 Dec 2019 19:13:10 +0100 Subject: [PATCH 020/138] [new-type] __openldap_server Originally developed at ungleich and improved for Open Sourcing by Evilham. --- cdist/conf/type/__openldap_server/man.rst | 148 +++++++++++ cdist/conf/type/__openldap_server/manifest | 235 ++++++++++++++++++ .../type/__openldap_server/parameter/boolean | 2 + .../type/__openldap_server/parameter/optional | 4 + .../parameter/optional_multiple | 2 + .../type/__openldap_server/parameter/required | 4 + 6 files changed, 395 insertions(+) create mode 100644 cdist/conf/type/__openldap_server/man.rst create mode 100644 cdist/conf/type/__openldap_server/manifest create mode 100644 cdist/conf/type/__openldap_server/parameter/boolean create mode 100644 cdist/conf/type/__openldap_server/parameter/optional create mode 100644 cdist/conf/type/__openldap_server/parameter/optional_multiple create mode 100644 cdist/conf/type/__openldap_server/parameter/required diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst new file mode 100644 index 00000000..29bbc231 --- /dev/null +++ b/cdist/conf/type/__openldap_server/man.rst @@ -0,0 +1,148 @@ +cdist-type__openldap_server(7) +============================== + +NAME +---- +cdist-type__openldap_server - Setup an openldap(4) server instance + + +DESCRIPTION +----------- +This type can be used to bootstrap an LDAP environment using openldap as slapd. + + +REQUIRED PARAMETERS +------------------- +manager-dn + The rootdn to set up in the directory. + E.g. `cn=manager,dc=ungleich,dc=ch`. See `slapd.conf(5)`. + +manager-password-hash + The password for `manager-dn` in the directory. + This should be valid for `slapd.conf` like `{SSHA}qV+mCs3u8Q2sCmUXT4Ybw7MebHTASMyr`. + Generate e.g. with: `slappasswd -s weneedgoodsecurity`. + See `slappasswd(8C)`, `slapd.conf(5)`. + TODO: implement this: http://blog.adamsbros.org/2015/06/09/openldap-ssha-salted-hashes-by-hand/ + to allow for a manager-password parameter and ensure idempotency (care with salts). + Such manager-password parameter should be mutually exclusive with this one. + +serverid + The server for the directory. + E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`. + +suffix + The suffix for the directory. + E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`. + + +OPTIONAL PARAMETERS +------------------- +syncrepl-credentials + Only has an effect if `replicate` is set; required in that case. + This secret is shared amongst the hosts that will replicate the directory. + Note that each replication server needs this secret and it is saved in + plain text in the directory. + +syncrepl-searchbase + Only has an effect if `replicate` is set; required in that case. + The searchbase to use for replication. + E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`. + +tls-cert + If defined, `__letsencrypt_cert` is not used and this must be the path in + the remote hosts to the PEM-encoded TLS certificate. + Requires: `tls-privkey` and `tls-ca`. + Permissions, existence and renewal of these files are left up to the + type's user. + +tls-privkey + Required if `tls-cert` is defined. + Path in the remote hosts to the PEM-encoded private key file. + +tls-ca + Required if `tls-cert` is defined. + Path in the remote hosts to the PEM-encoded CA certificate file. + + +OPTIONAL MULTIPLE PARAMETERS +---------------------------- +syncrepl-host + Only has an effect if `replicate` is set; required in that case. + Set once per host that will replicate the directory. + +module + LDAP module to load. See `slapd.conf(5)`. + Default value is OS-dependent, see manifest. + + +BOOLEAN PARAMETERS +------------------ +staging + Passed to `cdist-type__letsencrypt_cert`; has otherwise no use. + Obtain a test certificate from a staging server. + +replicate + Whether to setup replication or not. + If present `syncrepl-credentials` and `syncrepl-host` are also required. + +EXAMPLES +-------- + +.. code-block:: sh + + # Modify the ruleset on $__target_host: + __pf_ruleset --state present --source /my/pf/ruleset.conf + require="__pf_ruleset" \ + __pf_apply + + # Remove the ruleset on $__target_host (implies disabling pf(4): + __pf_ruleset --state absent + require="__pf_ruleset" \ + __pf_apply + + root@ldap-for-guacamole:~# cat ldapbase.ldif + dn: dc=guaca-test,dc=ungleich,dc=ch + objectClass: top + objectClass: dcObject + objectClass: organization + o: Some description + dc: guaca-test + + + # Sample usage: + # + # id=1 + # for host in ldap-test1.ungleich.ch ldap-test2.ungleich.ch; do + # echo "__ungleich_ldap ${host} \ + # --manager-dn 'cn=manager,dc=ungleich,dc=ch' \ + # --manager-password '{SSHA}fooo' \ + # --serverid '${id}' \ + # --staging \ + # --suffix 'dc=ungleich,dc=ch' \ + # --searchbase 'dc=ungleich,dc=ch' \ + # --syncrepl-credentials 'fooo' \ + # --syncrepl-host 'ldap-test1.ungleich.ch' \ + # --syncrepl-host 'ldap-test2.ungleich.ch' \ + # --descriptiont 'Ungleich LDAP server'" \ + # | cdist config -i - -v ${host} + # id=$((id + 1)) + # done + + +SEE ALSO +-------- +:strong:`cdist-type__letsencrypt_cert`\ (7) + + +AUTHORS +------- +ungleich +Evilham + + +COPYING +------- +Copyright \(C) 2020 ungleich glarus ag. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest new file mode 100644 index 00000000..2acaaed5 --- /dev/null +++ b/cdist/conf/type/__openldap_server/manifest @@ -0,0 +1,235 @@ +#!/bin/sh + +name="${__target_host}" +manager_dn=$(cat "${__object}/parameter/manager-dn") +manager_password_hash=$(cat "${__object}/parameter/manager-password-hash") +serverid=$(cat "${__object}/parameter/serverid") +suffix=$(cat "${__object}/parameter/suffix") +slapd_modules=$(cat "${__object}/parameter/module" || true) + + +OS="$(cat "${__global}/explorer/os")" + +# Setup OS-dependent vars +# TODO: treat other OS better, defaulting to Debian-like +case "${OS}" in + freebsd) + PKGS="openldap-server" + ETC="/usr/local/etc" + SLAPD_DIR="/usr/local/etc/openldap" + SLAPD_DATA_DIR="/var/db/openldap-data" + SLAPD_RUN_DIR="/var/run/openldap" + SLAPD_MODULE_PATH="/usr/local/libexec/openldap" + if [ -z "${slapd_modules}" ]; then + # It looks like ppolicy and syncprov must be compiled + slapd_modules="back_mdb back_monitor" + fi + ;; + *) + PKGS="slapd ldap-utils" + ETC="/etc" + SLAPD_DIR="/etc/ldap" + SLAPD_DATA_DIR="/var/lib/ldap" + SLAPD_RUN_DIR="/var/run/slapd" + SLAPD_MODULE_PATH="/usr/lib/ldap" + if [ -z "${slapd_modules}" ]; then + slapd_modules="back_mdb ppolicy syncprov back_monitor" + fi + ;; +esac + + + +# Determine if __letsencrypt_cert is to be used and setup vars accordingly +if [ -f "${__object}/parameter/tls-cert" ]; then + tls_cert=$(cat "${__object}/parameter/tls-cert") + + if [ ! -f "${__object}/parameter/tls-privkey" ]; then + echo "When tls-cert is defined, tls-privkey is also required." >&2 + exit 1 + fi + tls_privkey=$(cat "${__object}/parameter/tls-privkey") + + if [ ! -f "${__object}/parameter/tls-ca" ]; then + echo "When tls-cert is defined, tls-ca is also required." >&2 + exit 1 + fi + tls_ca=$(cat "${__object}/parameter/tls-ca") + + _skip_letsencrypt_cert="YES" +else + tls_cert="${SLAPD_DIR}/sasl2/cert.pem" + tls_privkey="${SLAPD_DIR}/sasl2/privkey.pem" + tls_ca="${SLAPD_DIR}/sasl2/chain.pem" +fi + +mkdir "${__object}/files" +ldapconf="${__object}/files/ldapconf" + +replication="" +if [ -f "${__object}/parameter/replicate" ]; then + replication=yes + + if [ ! -f "${__object}/parameter/syncrepl-searchbase" ]; then + echo "Requiring the searchbase for replication" >&2 + exit 1 + fi + syncrepl_searchbase=$(cat "${__object}/parameter/syncrepl-searchbase") + + if [ ! -f "${__object}/parameter/syncrepl-credentials" ]; then + echo "Requiring credentials for replication" >&2 + exit 1 + fi + + syncrepl_credentials=$(cat "${__object}/parameter/syncrepl-credentials") + + if [ ! -f "${__object}/parameter/syncrepl-host" ]; then + echo "Requiring host(s) for replication" >&2 + exit 1 + fi + syncrepl_hosts=$(cat "${__object}/parameter/syncrepl-host") + +fi + +# Install required packages +for pkg in ${PKGS}; do + __package ${pkg} +done + + +# TODO: Implement __start_on_boot for BSD +require="__package/slapd" __start_on_boot slapd + +# TODO: treat other OS better. Defaulting to Debian-like. +if [ "${OS}" != "freebsd" ]; then + require="__package/slapd" __line rm_slapd_conf \ + --file ${ETC}/default/slapd \ + --regex 'SLAPD_CONF=.*' \ + --state absent + + require="__package/slapd" __line rm_slapd_services \ + --file ${ETC}/default/slapd \ + --regex 'SLAPD_SERVICES=.*' \ + --state absent + + require="__line/rm_slapd_conf" __line add_slapd_conf \ + --file ${ETC}/default/slapd \ + --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \ + --state present + + require="__line/rm_slapd_services" __line add_slapd_services \ + --file ${ETC}/default/slapd \ + --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \ + --state present +fi + + +if [ -z "${_skip_letsencrypt_cert}" ]; then + if [ -f "${__object}/parameter/staging" ]; then + staging="--staging" + else + staging="" + fi + + __letsencrypt_cert "${name}" --admin-email technik@ungleich.ch \ + --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \ + --automatic-renewal ${staging} +fi + +require="__package/slapd" __directory ${SLAPD_DIR}/slapd.d --state absent + +if [ -z "${_skip_letsencrypt_cert}" ]; then + require="__package/slapd __letsencrypt_cert/${name}" \ + __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \ + --source "${ldapconf}" +else + require="__package/slapd" \ + __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \ + --source "${ldapconf}" +fi + +# Start slapd.conf +cat << EOF > "${ldapconf}" +pidfile ${SLAPD_RUN_DIR}/slapd.pid +argsfile ${SLAPD_RUN_DIR}/slapd.args + +TLSCipherSuite NORMAL +TLSCertificateFile ${tls_cert} +TLSCertificateKeyFile ${tls_privkey} +TLSCACertificateFile ${tls_ca} + +disallow bind_anon +require bind +security tls=1 + +include ${SLAPD_DIR}/schema/corba.schema +include ${SLAPD_DIR}/schema/core.schema +include ${SLAPD_DIR}/schema/cosine.schema +include ${SLAPD_DIR}/schema/duaconf.schema +include ${SLAPD_DIR}/schema/dyngroup.schema +include ${SLAPD_DIR}/schema/inetorgperson.schema +include ${SLAPD_DIR}/schema/java.schema +include ${SLAPD_DIR}/schema/misc.schema +include ${SLAPD_DIR}/schema/nis.schema +include ${SLAPD_DIR}/schema/openldap.schema +include ${SLAPD_DIR}/schema/ppolicy.schema +include ${SLAPD_DIR}/schema/collective.schema + +modulepath ${SLAPD_MODULE_PATH} +EOF + +# Add specified modules +for module in ${slapd_modules}; do + echo "moduleload ${module}.la" >> "${ldapconf}" +done + +# Rest of the config +cat << EOF >> "${ldapconf}" +loglevel 1024 + +database mdb +maxsize 1073741824 + +suffix "${suffix}" +directory ${SLAPD_DATA_DIR} +rootdn "${manager_dn}" +rootpw "${manager_password_hash}" + +index objectClass eq,pres +index ou,cn,mail,surname,givenname eq,pres,sub +index uidNumber,gidNumber,loginShell eq,pres +index uid,memberUid eq,pres,sub +index nisMapName,nisMapEntry eq,pres,sub +index entryCSN,entryUUID eq + +serverid ${serverid} +EOF + +# Setup replication +if [ "${replication}" ]; then + rid=1; + for syncrepl in ${syncrepl_hosts}; do + cat <> "${ldapconf}" +syncrepl rid=${rid} + provider=ldap://${syncrepl} + bindmethod=simple + starttls=yes + binddn="${manager_dn}" + credentials=${syncrepl_credentials} + searchbase="${syncrepl_searchbase}" + type=refreshAndPersist + retry="5 + 5 +" + interval=00:00:00:05 +EOF + rid=$((rid + 1)) + done + cat <> "${ldapconf}" +mirrormode true +overlay syncprov +syncprov-checkpoint 100 5 +syncprov-sessionlog 100 + +database monitor +limits dn.exact="${manager_dn}" time=unlimited size=unlimited +EOF +fi diff --git a/cdist/conf/type/__openldap_server/parameter/boolean b/cdist/conf/type/__openldap_server/parameter/boolean new file mode 100644 index 00000000..45056fe9 --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/boolean @@ -0,0 +1,2 @@ +staging +replicate diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional new file mode 100644 index 00000000..a9a8ab2c --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/optional @@ -0,0 +1,4 @@ +description +syncrepl-credentials +syncrepl-searchbase +tls-cert diff --git a/cdist/conf/type/__openldap_server/parameter/optional_multiple b/cdist/conf/type/__openldap_server/parameter/optional_multiple new file mode 100644 index 00000000..107c03d9 --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/optional_multiple @@ -0,0 +1,2 @@ +syncrepl-host +module diff --git a/cdist/conf/type/__openldap_server/parameter/required b/cdist/conf/type/__openldap_server/parameter/required new file mode 100644 index 00000000..1ee6f219 --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/required @@ -0,0 +1,4 @@ +manager-dn +manager-password-hash +serverid +suffix From fd430eab622206b9a04780ca5fa9d7c807f16c93 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 9 Dec 2019 19:39:43 +0100 Subject: [PATCH 021/138] [new-type] __openldap_server: Add a "schema" optional parameter. --- cdist/conf/type/__openldap_server/man.rst | 7 +++++ cdist/conf/type/__openldap_server/manifest | 29 +++++++------------ .../parameter/default/schema | 12 ++++++++ .../type/__openldap_server/parameter/optional | 3 +- 4 files changed, 31 insertions(+), 20 deletions(-) create mode 100644 cdist/conf/type/__openldap_server/parameter/default/schema diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst index 29bbc231..1fc24eaa 100644 --- a/cdist/conf/type/__openldap_server/man.rst +++ b/cdist/conf/type/__openldap_server/man.rst @@ -74,6 +74,13 @@ module LDAP module to load. See `slapd.conf(5)`. Default value is OS-dependent, see manifest. +schema + Name of LDAP schema to load. Must be the name without extension of a + `.schema` file in slapd's schema directory (usually `/etc/slapd/schema` or + `/usr/local/etc/openldap/schema`). + Example value: `inetorgperson` + The type user must ensure that the schema file is deployed. + This defaults to a sensible subset, for details see the type definition. BOOLEAN PARAMETERS ------------------ diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest index 2acaaed5..518edd86 100644 --- a/cdist/conf/type/__openldap_server/manifest +++ b/cdist/conf/type/__openldap_server/manifest @@ -6,6 +6,7 @@ manager_password_hash=$(cat "${__object}/parameter/manager-password-hash") serverid=$(cat "${__object}/parameter/serverid") suffix=$(cat "${__object}/parameter/suffix") slapd_modules=$(cat "${__object}/parameter/module" || true) +schemas=$(cat "${__object}/parameter/schema") OS="$(cat "${__global}/explorer/os")" @@ -14,8 +15,8 @@ OS="$(cat "${__global}/explorer/os")" # TODO: treat other OS better, defaulting to Debian-like case "${OS}" in freebsd) - PKGS="openldap-server" - ETC="/usr/local/etc" + PKGS="openldap-server" + ETC="/usr/local/etc" SLAPD_DIR="/usr/local/etc/openldap" SLAPD_DATA_DIR="/var/db/openldap-data" SLAPD_RUN_DIR="/var/run/openldap" @@ -27,7 +28,7 @@ case "${OS}" in ;; *) PKGS="slapd ldap-utils" - ETC="/etc" + ETC="/etc" SLAPD_DIR="/etc/ldap" SLAPD_DATA_DIR="/var/lib/ldap" SLAPD_RUN_DIR="/var/run/slapd" @@ -39,7 +40,6 @@ case "${OS}" in esac - # Determine if __letsencrypt_cert is to be used and setup vars accordingly if [ -f "${__object}/parameter/tls-cert" ]; then tls_cert=$(cat "${__object}/parameter/tls-cert") @@ -161,24 +161,15 @@ TLSCACertificateFile ${tls_ca} disallow bind_anon require bind security tls=1 - -include ${SLAPD_DIR}/schema/corba.schema -include ${SLAPD_DIR}/schema/core.schema -include ${SLAPD_DIR}/schema/cosine.schema -include ${SLAPD_DIR}/schema/duaconf.schema -include ${SLAPD_DIR}/schema/dyngroup.schema -include ${SLAPD_DIR}/schema/inetorgperson.schema -include ${SLAPD_DIR}/schema/java.schema -include ${SLAPD_DIR}/schema/misc.schema -include ${SLAPD_DIR}/schema/nis.schema -include ${SLAPD_DIR}/schema/openldap.schema -include ${SLAPD_DIR}/schema/ppolicy.schema -include ${SLAPD_DIR}/schema/collective.schema - -modulepath ${SLAPD_MODULE_PATH} EOF +# Add specified schemas +for schema in ${schemas}; do + echo "include ${SLAPD_DIR}/schema/${schema}.schema" >> "${ldapconf}" +done + # Add specified modules +echo "modulepath ${SLAPD_MODULE_PATH}" >> "${ldapconf}" for module in ${slapd_modules}; do echo "moduleload ${module}.la" >> "${ldapconf}" done diff --git a/cdist/conf/type/__openldap_server/parameter/default/schema b/cdist/conf/type/__openldap_server/parameter/default/schema new file mode 100644 index 00000000..825bdb15 --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/default/schema @@ -0,0 +1,12 @@ +corba +core +cosine +duaconf +dyngroup +inetorgperson +java +misc +nis +openldap +ppolicy +collective diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional index a9a8ab2c..53587c4e 100644 --- a/cdist/conf/type/__openldap_server/parameter/optional +++ b/cdist/conf/type/__openldap_server/parameter/optional @@ -1,4 +1,5 @@ -description syncrepl-credentials syncrepl-searchbase tls-cert +tls-privkey +tls-ca From 1ad605a509cfd534334111c1d6c1ea8682446b10 Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 9 Dec 2019 19:49:05 +0100 Subject: [PATCH 022/138] [new-type] __openldap_server: Add admin-email parameter. --- cdist/conf/type/__openldap_server/man.rst | 5 +++++ cdist/conf/type/__openldap_server/manifest | 8 +++++++- cdist/conf/type/__openldap_server/parameter/optional | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst index 1fc24eaa..4447971a 100644 --- a/cdist/conf/type/__openldap_server/man.rst +++ b/cdist/conf/type/__openldap_server/man.rst @@ -48,6 +48,11 @@ syncrepl-searchbase The searchbase to use for replication. E.g. `dc=ungleich,dc=ch`. See `slapd.conf(5)`. +admin-email + Passed to `cdist-type__letsencrypt_cert`; has otherwise no use. + Required if using `__letsencrypt_cert`. + Where to send Let's Encrypt emails like "certificate needs renewal". + tls-cert If defined, `__letsencrypt_cert` is not used and this must be the path in the remote hosts to the PEM-encoded TLS certificate. diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest index 518edd86..e10432d6 100644 --- a/cdist/conf/type/__openldap_server/manifest +++ b/cdist/conf/type/__openldap_server/manifest @@ -58,6 +58,12 @@ if [ -f "${__object}/parameter/tls-cert" ]; then _skip_letsencrypt_cert="YES" else + if [ ! -f "${__object}/parameter/admin-email" ]; then + echo "When using __letsencrypt_cert, admin-email is also required." >&2 + exit 1 + fi + admin_email=$(cat "${__object}/parameter/admin-email") + tls_cert="${SLAPD_DIR}/sasl2/cert.pem" tls_privkey="${SLAPD_DIR}/sasl2/privkey.pem" tls_ca="${SLAPD_DIR}/sasl2/chain.pem" @@ -131,7 +137,7 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then staging="" fi - __letsencrypt_cert "${name}" --admin-email technik@ungleich.ch \ + __letsencrypt_cert "${name}" --admin-email "${admin_email}" \ --renew-hook "cp ${ETC}/letsencrypt/live/${name}/*.pem ${SLAPD_DIR}/sasl2 && chown -R openldap:openldap ${SLAPD_DIR}/sasl2 && service slapd restart" \ --automatic-renewal ${staging} fi diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional index 53587c4e..f4254cb6 100644 --- a/cdist/conf/type/__openldap_server/parameter/optional +++ b/cdist/conf/type/__openldap_server/parameter/optional @@ -1,5 +1,6 @@ syncrepl-credentials syncrepl-searchbase +admin-email tls-cert tls-privkey tls-ca From 42914d26c5addae027cc382ec035a687cd3aaf6d Mon Sep 17 00:00:00 2001 From: Evilham Date: Mon, 9 Dec 2019 19:59:15 +0100 Subject: [PATCH 023/138] [new-type] __openldap_server: sudo make nico.happy. --- cdist/conf/type/__openldap_server/manifest | 53 ++++++++++++---------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest index e10432d6..ffd92626 100644 --- a/cdist/conf/type/__openldap_server/manifest +++ b/cdist/conf/type/__openldap_server/manifest @@ -9,11 +9,10 @@ slapd_modules=$(cat "${__object}/parameter/module" || true) schemas=$(cat "${__object}/parameter/schema") -OS="$(cat "${__global}/explorer/os")" +os="$(cat "${__global}/explorer/os")" # Setup OS-dependent vars -# TODO: treat other OS better, defaulting to Debian-like -case "${OS}" in +case "${os}" in freebsd) PKGS="openldap-server" ETC="/usr/local/etc" @@ -26,7 +25,7 @@ case "${OS}" in slapd_modules="back_mdb back_monitor" fi ;; - *) + debian|ubuntu|devuan) PKGS="slapd ldap-utils" ETC="/etc" SLAPD_DIR="/etc/ldap" @@ -37,6 +36,10 @@ case "${OS}" in slapd_modules="back_mdb ppolicy syncprov back_monitor" fi ;; + *) + echo "Don't know the openldap defaults for: $os" >&2 + exit 1 + ;; esac @@ -106,28 +109,32 @@ done # TODO: Implement __start_on_boot for BSD require="__package/slapd" __start_on_boot slapd -# TODO: treat other OS better. Defaulting to Debian-like. -if [ "${OS}" != "freebsd" ]; then - require="__package/slapd" __line rm_slapd_conf \ - --file ${ETC}/default/slapd \ - --regex 'SLAPD_CONF=.*' \ - --state absent +case "${os}" in + debian|ubuntu|devuan) + require="__package/slapd" __line rm_slapd_conf \ + --file ${ETC}/default/slapd \ + --regex 'SLAPD_CONF=.*' \ + --state absent - require="__package/slapd" __line rm_slapd_services \ - --file ${ETC}/default/slapd \ - --regex 'SLAPD_SERVICES=.*' \ - --state absent + require="__package/slapd" __line rm_slapd_services \ + --file ${ETC}/default/slapd \ + --regex 'SLAPD_SERVICES=.*' \ + --state absent - require="__line/rm_slapd_conf" __line add_slapd_conf \ - --file ${ETC}/default/slapd \ - --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \ - --state present + require="__line/rm_slapd_conf" __line add_slapd_conf \ + --file ${ETC}/default/slapd \ + --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \ + --state present - require="__line/rm_slapd_services" __line add_slapd_services \ - --file ${ETC}/default/slapd \ - --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \ - --state present -fi + require="__line/rm_slapd_services" __line add_slapd_services \ + --file ${ETC}/default/slapd \ + --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \ + --state present + ;; + *) + # Nothing to do here, move on. + ;; +esac if [ -z "${_skip_letsencrypt_cert}" ]; then From 22c5cd550bb1069c2bb8106bbe907f016b9b821a Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 10 Dec 2019 12:49:07 +0100 Subject: [PATCH 024/138] [new-type] __openldap_server: first public version. This already takes care of setting up the base DN and managing it as well as allowing for settings for the listener URLS. The type was also made a singleton as it doesn't make much sense to setup multiple LDAP servers on the same machine. --- .../type/__openldap_server/gencode-remote | 44 ++++++ cdist/conf/type/__openldap_server/man.rst | 126 +++++++++++++----- cdist/conf/type/__openldap_server/manifest | 48 +++++-- .../parameter/default/description | 1 + .../type/__openldap_server/parameter/optional | 4 +- .../parameter/optional_multiple | 1 + .../type/__openldap_server/parameter/required | 1 + .../parameter/required_multiple | 1 + cdist/conf/type/__openldap_server/singleton | 0 9 files changed, 176 insertions(+), 50 deletions(-) create mode 100644 cdist/conf/type/__openldap_server/gencode-remote create mode 100644 cdist/conf/type/__openldap_server/parameter/default/description create mode 100644 cdist/conf/type/__openldap_server/parameter/required_multiple create mode 100644 cdist/conf/type/__openldap_server/singleton diff --git a/cdist/conf/type/__openldap_server/gencode-remote b/cdist/conf/type/__openldap_server/gencode-remote new file mode 100644 index 00000000..0ac434af --- /dev/null +++ b/cdist/conf/type/__openldap_server/gencode-remote @@ -0,0 +1,44 @@ +#!/bin/sh + +manager_dn=$(cat "${__object}/parameter/manager-dn") +manager_password=$(cat "${__object}/parameter/manager-password") +description=$(cat "${__object}/parameter/description") +suffix=$(cat "${__object}/parameter/suffix") +suffix_dc=$(echo -n ${suffix} | awk -F',' '{print $1}' | awk -F'=' '{print $2}') + +SLAPD_IPC=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ' | awk '{ print $1}') + +cat <&1 > /dev/null; then + # Already exists, use ldapmodify + ldapmodify -xZ -D "${manager_dn}" -w "${manager_password}" -H '${SLAPD_IPC}' </dev/null || true) schemas=$(cat "${__object}/parameter/schema") +slapd_urls=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ') +tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true) os="$(cat "${__global}/explorer/os")" # Setup OS-dependent vars +CONF_OWNER="root" +CONF_GROUP="root" case "${os}" in freebsd) PKGS="openldap-server" @@ -24,6 +28,12 @@ case "${os}" in # It looks like ppolicy and syncprov must be compiled slapd_modules="back_mdb back_monitor" fi + CONF_OWNER="ldap" + CONF_GROUP="ldap" + if [ -z "${tls_cipher_suite}" ]; then + # TODO: research default for FreeBSD. 'NORMAL' appears to not work + tls_cipher_suite="HIGH:MEDIUM:+SSLv2" + fi ;; debian|ubuntu|devuan) PKGS="slapd ldap-utils" @@ -35,6 +45,9 @@ case "${os}" in if [ -z "${slapd_modules}" ]; then slapd_modules="back_mdb ppolicy syncprov back_monitor" fi + if [ -z "${tls_cipher_suite}" ]; then + tls_cipher_suite="NORMAL" + fi ;; *) echo "Don't know the openldap defaults for: $os" >&2 @@ -42,6 +55,8 @@ case "${os}" in ;; esac +PKG_MAIN=$(echo ${PKGS} | awk '{print $1;}') + # Determine if __letsencrypt_cert is to be used and setup vars accordingly if [ -f "${__object}/parameter/tls-cert" ]; then @@ -106,17 +121,26 @@ for pkg in ${PKGS}; do done -# TODO: Implement __start_on_boot for BSD -require="__package/slapd" __start_on_boot slapd +require="__package/${PKG_MAIN}" __start_on_boot slapd +# Setup -h flag for the listeners. See man slapd (-h flag). case "${os}" in + freebsd) + require="__package/${PKG_MAIN}" __key_value \ + --file "/etc/rc.conf" \ + --key "slapd_flags" \ + --value "\"-h '${slapd_urls}'\"" \ + --delimiter "=" \ + --comment "# LDAP Listener URLs" \ + "${__target_host}__slapd_flags" + ;; debian|ubuntu|devuan) - require="__package/slapd" __line rm_slapd_conf \ + require="__package/${PKG_MAIN}" __line rm_slapd_conf \ --file ${ETC}/default/slapd \ --regex 'SLAPD_CONF=.*' \ --state absent - require="__package/slapd" __line rm_slapd_services \ + require="__package/${PKG_MAIN}" __line rm_slapd_services \ --file ${ETC}/default/slapd \ --regex 'SLAPD_SERVICES=.*' \ --state absent @@ -128,7 +152,7 @@ case "${os}" in require="__line/rm_slapd_services" __line add_slapd_services \ --file ${ETC}/default/slapd \ - --line "SLAPD_SERVICES=\"ldap://localhost/ ldap://${name}/\"" \ + --line "SLAPD_SERVICES=\"${slapd_urls}\"" \ --state present ;; *) @@ -149,15 +173,15 @@ if [ -z "${_skip_letsencrypt_cert}" ]; then --automatic-renewal ${staging} fi -require="__package/slapd" __directory ${SLAPD_DIR}/slapd.d --state absent +require="__package/${PKG_MAIN}" __directory ${SLAPD_DIR}/slapd.d --state absent if [ -z "${_skip_letsencrypt_cert}" ]; then - require="__package/slapd __letsencrypt_cert/${name}" \ - __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \ + require="__package/${PKG_MAIN} __letsencrypt_cert/${name}" \ + __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \ --source "${ldapconf}" else - require="__package/slapd" \ - __file ${SLAPD_DIR}/slapd.conf --owner root --group root --mode 644 \ + require="__package/${PKG_MAIN}" \ + __file ${SLAPD_DIR}/slapd.conf --owner ${CONF_OWNER} --group ${CONF_GROUP} --mode 644 \ --source "${ldapconf}" fi @@ -166,7 +190,7 @@ cat << EOF > "${ldapconf}" pidfile ${SLAPD_RUN_DIR}/slapd.pid argsfile ${SLAPD_RUN_DIR}/slapd.args -TLSCipherSuite NORMAL +TLSCipherSuite ${tls_cipher_suite} TLSCertificateFile ${tls_cert} TLSCertificateKeyFile ${tls_privkey} TLSCACertificateFile ${tls_ca} diff --git a/cdist/conf/type/__openldap_server/parameter/default/description b/cdist/conf/type/__openldap_server/parameter/default/description new file mode 100644 index 00000000..6d8e37e1 --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/default/description @@ -0,0 +1 @@ +Managed by cdist, do not edit manually. diff --git a/cdist/conf/type/__openldap_server/parameter/optional b/cdist/conf/type/__openldap_server/parameter/optional index f4254cb6..a92b9c6e 100644 --- a/cdist/conf/type/__openldap_server/parameter/optional +++ b/cdist/conf/type/__openldap_server/parameter/optional @@ -1,6 +1,8 @@ +description syncrepl-credentials syncrepl-searchbase admin-email +tls-cipher-suite tls-cert tls-privkey -tls-ca +tls-ca \ No newline at end of file diff --git a/cdist/conf/type/__openldap_server/parameter/optional_multiple b/cdist/conf/type/__openldap_server/parameter/optional_multiple index 107c03d9..52a83d5c 100644 --- a/cdist/conf/type/__openldap_server/parameter/optional_multiple +++ b/cdist/conf/type/__openldap_server/parameter/optional_multiple @@ -1,2 +1,3 @@ syncrepl-host module +schema diff --git a/cdist/conf/type/__openldap_server/parameter/required b/cdist/conf/type/__openldap_server/parameter/required index 1ee6f219..ff58158d 100644 --- a/cdist/conf/type/__openldap_server/parameter/required +++ b/cdist/conf/type/__openldap_server/parameter/required @@ -1,4 +1,5 @@ manager-dn +manager-password manager-password-hash serverid suffix diff --git a/cdist/conf/type/__openldap_server/parameter/required_multiple b/cdist/conf/type/__openldap_server/parameter/required_multiple new file mode 100644 index 00000000..848b8dc2 --- /dev/null +++ b/cdist/conf/type/__openldap_server/parameter/required_multiple @@ -0,0 +1 @@ +slapd-url \ No newline at end of file diff --git a/cdist/conf/type/__openldap_server/singleton b/cdist/conf/type/__openldap_server/singleton new file mode 100644 index 00000000..e69de29b From 79d58f0813fd08c77d826ac552dfdb9e0f40dbbd Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 10 Dec 2019 13:04:24 +0100 Subject: [PATCH 025/138] [new-type] __openldap_server: fix docs / dependency. It's nicer to have slapd_flags after slapd_enable in rc.conf on freebsd. --- cdist/conf/type/__openldap_server/man.rst | 6 +++--- cdist/conf/type/__openldap_server/manifest | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst index 84b534bd..1c2c9b70 100644 --- a/cdist/conf/type/__openldap_server/man.rst +++ b/cdist/conf/type/__openldap_server/man.rst @@ -141,9 +141,9 @@ EXAMPLES --serverid 0 \ --suffix 'dc=camilion,dc=cloud' \ --slapd-url 'ldaps://ldap.camilion.cloud' \ - --tls-cert '${pki_prefix}/default.crt' \ - --tls-privkey '${pki_prefix}/default.key' \ - --tls-ca '${pki_prefix}/CA.crt' + --tls-cert "${pki_prefix}/default.crt" \ + --tls-privkey "${pki_prefix}/default.key" \ + --tls-ca "${pki_prefix}/CA.crt" # The created basedn looks as follows: # diff --git a/cdist/conf/type/__openldap_server/manifest b/cdist/conf/type/__openldap_server/manifest index 2d1df32f..070a31d5 100644 --- a/cdist/conf/type/__openldap_server/manifest +++ b/cdist/conf/type/__openldap_server/manifest @@ -126,7 +126,7 @@ require="__package/${PKG_MAIN}" __start_on_boot slapd # Setup -h flag for the listeners. See man slapd (-h flag). case "${os}" in freebsd) - require="__package/${PKG_MAIN}" __key_value \ + require="__start_on_boot/slapd" __key_value \ --file "/etc/rc.conf" \ --key "slapd_flags" \ --value "\"-h '${slapd_urls}'\"" \ From 35f0d4dbfa9307923fa013ff806d30087a8cf291 Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 10 Dec 2019 13:10:19 +0100 Subject: [PATCH 026/138] [new-type] __openldap_server: another documentation typo. --- cdist/conf/type/__openldap_server/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__openldap_server/man.rst b/cdist/conf/type/__openldap_server/man.rst index 1c2c9b70..d20101d1 100644 --- a/cdist/conf/type/__openldap_server/man.rst +++ b/cdist/conf/type/__openldap_server/man.rst @@ -133,8 +133,8 @@ EXAMPLES .. code-block:: sh # Example of a simple server with manual certificate management. - pki_prefix="/usr/local/etc/pki/realms/ldap.camilion.cloud" \ - __openldap_server \ + pki_prefix="/usr/local/etc/pki/realms/ldap.camilion.cloud" + __openldap_server \ --manager-dn 'cn=manager,dc=camilion,dc=cloud' \ --manager-password "foo" \ --manager-password-hash '{SSHA}foo' \ From 3ba230c10db98cf2cddb0aeeb256392223fb2558 Mon Sep 17 00:00:00 2001 From: Evilham Date: Tue, 10 Dec 2019 13:12:54 +0100 Subject: [PATCH 027/138] [type-docs] __start_on_boot: remove unspported *BSD claim. The type appears to support {Open,Free}BSD properly. --- cdist/conf/type/__start_on_boot/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__start_on_boot/man.rst b/cdist/conf/type/__start_on_boot/man.rst index b7c73ab1..f8afe94b 100644 --- a/cdist/conf/type/__start_on_boot/man.rst +++ b/cdist/conf/type/__start_on_boot/man.rst @@ -12,7 +12,7 @@ This cdist type allows you to enable or disable stuff to be started at boot of your operating system. Warning: This type has not been tested intensively and is not fully -supported (i.e. \*BSD are not implemented). +supported. REQUIRED PARAMETERS From 3c62a88ca7907b15dc501ddb3395c66b6196cb13 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 10 Dec 2019 19:33:29 +0100 Subject: [PATCH 028/138] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 04ffac62..186dce7a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,8 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) + * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) + * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 * Core: Redefine/reimplement/fix CDIST_ORDER_DEPENDENCY (Darko Poljak) From 2c7b4ddc55a9a2f76ae5b51a6020c0cd14900705 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 12 Dec 2019 06:18:04 +0100 Subject: [PATCH 029/138] Update cdist man page --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index d6937272..55db82ed 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -915,5 +915,5 @@ such case and display a warning message. An example of such a case: COPYING ------- -Copyright \(C) 2011-2017 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License v3 or later (GPLv3+). From bd4eee7925356c8e557a75688694feff94cca297 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 12 Dec 2019 07:00:23 +0100 Subject: [PATCH 030/138] Fix shellcheck reported issues --- cdist/conf/type/__openldap_server/gencode-remote | 4 ++-- cdist/conf/type/__openldap_server/manifest | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__openldap_server/gencode-remote b/cdist/conf/type/__openldap_server/gencode-remote index 0ac434af..b1e98f8c 100644 --- a/cdist/conf/type/__openldap_server/gencode-remote +++ b/cdist/conf/type/__openldap_server/gencode-remote @@ -4,9 +4,9 @@ manager_dn=$(cat "${__object}/parameter/manager-dn") manager_password=$(cat "${__object}/parameter/manager-password") description=$(cat "${__object}/parameter/description") suffix=$(cat "${__object}/parameter/suffix") -suffix_dc=$(echo -n ${suffix} | awk -F',' '{print $1}' | awk -F'=' '{print $2}') +suffix_dc=$(printf "%s" "${suffix}" | awk -F',' '{print $1}' | awk -F'=' '{print $2}') -SLAPD_IPC=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ' | awk '{ print $1}') +SLAPD_IPC=$(tr '\n' ' ' < "${__object}/parameter/slapd-url" | awk '{ print $1}') cat </dev/null || true) schemas=$(cat "${__object}/parameter/schema") -slapd_urls=$(cat "${__object}/parameter/slapd-url" | tr '\n' ' ') +slapd_urls=$(tr '\n' ' ' < "${__object}/parameter/slapd-url") tls_cipher_suite=$(cat "${__object}/parameter/tls-cipher-suite" 2>/dev/null || true) @@ -55,7 +55,7 @@ case "${os}" in ;; esac -PKG_MAIN=$(echo ${PKGS} | awk '{print $1;}') +PKG_MAIN=$(echo "${PKGS}" | awk '{print $1;}') # Determine if __letsencrypt_cert is to be used and setup vars accordingly @@ -117,7 +117,7 @@ fi # Install required packages for pkg in ${PKGS}; do - __package ${pkg} + __package "${pkg}" done @@ -147,7 +147,7 @@ case "${os}" in require="__line/rm_slapd_conf" __line add_slapd_conf \ --file ${ETC}/default/slapd \ - --line 'SLAPD_CONF=${SLAPD_DIR}/slapd.conf' \ + --line "SLAPD_CONF=${SLAPD_DIR}/slapd.conf" \ --state present require="__line/rm_slapd_services" __line add_slapd_services \ From 8562871da9a51752fea47308e496e139210edf75 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 12 Dec 2019 07:35:53 +0100 Subject: [PATCH 031/138] Fix shellcheck exit status shellcheck* targets were always reporting exit status 0. With this fix, if shellcheck fails, then build-helper script exits with 1. --- bin/build-helper | 50 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/bin/build-helper b/bin/build-helper index 7cfc4d55..69dee4c7 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -74,6 +74,7 @@ SHELLCHECKCMD="shellcheck -s sh -f gcc -x" # Skip SC2154 for variables starting with __ since such variables are cdist # environment variables. SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]' +SHELLCHECKTMP=".shellcheck.tmp" # Change to checkout directory basedir="${0%/*}/../" @@ -431,53 +432,67 @@ eof ;; shellcheck-global-explorers) - find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + # shellcheck disable=SC2086 + find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-type-explorers) - find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + # shellcheck disable=SC2086 + find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-manifests) - find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + # shellcheck disable=SC2086 + find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-local-gencodes) - find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + # shellcheck disable=SC2086 + find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-remote-gencodes) - find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + # shellcheck disable=SC2086 + find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-scripts) - ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0 + # shellcheck disable=SC2086 + ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-gencodes) - "$0" shellcheck-local-gencodes - "$0" shellcheck-remote-gencodes + "$0" shellcheck-local-gencodes || exit 1 + "$0" shellcheck-remote-gencodes || exit 1 ;; shellcheck-types) - "$0" shellcheck-type-explorers - "$0" shellcheck-manifests - "$0" shellcheck-gencodes + "$0" shellcheck-type-explorers || exit 1 + "$0" shellcheck-manifests || exit 1 + "$0" shellcheck-gencodes || exit 1 ;; shellcheck) - "$0" shellcheck-global-explorers - "$0" shellcheck-types - "$0" shellcheck-scripts + "$0" shellcheck-global-explorers || exit 1 + "$0" shellcheck-types || exit 1 + "$0" shellcheck-scripts || exit 1 ;; shellcheck-type-files) - find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + # shellcheck disable=SC2086 + find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" > "${SHELLCHECKTMP}" + test ! -s "${SHELLCHECKTMP}" || { cat "${SHELLCHECKTMP}"; exit 1; } ;; shellcheck-with-files) - "$0" shellcheck - "$0" shellcheck-type-files + "$0" shellcheck || exit 1 + "$0" shellcheck-type-files || exit 1 ;; shellcheck-build-helper) @@ -535,6 +550,7 @@ eof # Temp files rm -f ./*.tmp + rm -f ./.*.tmp ;; distclean) From 9859080217a9dc067250a0fc0f686a2242ef1083 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 12 Dec 2019 07:43:07 +0100 Subject: [PATCH 032/138] Release 6.3.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 186dce7a..674e14c7 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.3.0: 2019-12-12 * Type __package_update_index: Fix Alpine part (Dominique Roux) * Documentation: Fix man pages for install types (Darko Poljak) * Documentation: Embed config skeleton instead of rewriting it (Darko Poljak) From f407e8825fc2eb70fa489dc218b46f1f2d92d747 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 15 Dec 2019 22:56:16 +0100 Subject: [PATCH 033/138] [consul/alpine] no init script required to be deployed anymore --- cdist/conf/type/__consul_agent/manifest | 2 +- docs/changelog | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index a88d26ed..ee682d72 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -181,7 +181,7 @@ init_upstart() # Install init script to start on boot case "$os" in - alpine|devuan) + devuan) init_sysvinit debian ;; centos|redhat) diff --git a/docs/changelog b/docs/changelog index 674e14c7..415af0af 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * Type __consul_agent: don't deploy init script on Alpine anymore, + it ships with one itself (Nico Schottelius) + 6.3.0: 2019-12-12 * Type __package_update_index: Fix Alpine part (Dominique Roux) * Documentation: Fix man pages for install types (Darko Poljak) From df05abd15b6676d6594b3e8aebff112c2f5c0ce7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 19 Dec 2019 12:33:47 +0100 Subject: [PATCH 034/138] bugfix: __install_chroot_umount was not using __chroot_umount/manifest Signed-off-by: Steven Armstrong --- cdist/conf/type/__install_chroot_umount/manifest | 1 + 1 file changed, 1 insertion(+) create mode 120000 cdist/conf/type/__install_chroot_umount/manifest diff --git a/cdist/conf/type/__install_chroot_umount/manifest b/cdist/conf/type/__install_chroot_umount/manifest new file mode 120000 index 00000000..f17af67a --- /dev/null +++ b/cdist/conf/type/__install_chroot_umount/manifest @@ -0,0 +1 @@ +../__chroot_umount/manifest \ No newline at end of file From beb930c0dc6e80d0718029fa8fc5539f373154a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 31 Dec 2019 11:05:40 +0100 Subject: [PATCH 035/138] __postgres_*: use delimited identifiers (double quoted) in generated SQL --- cdist/conf/type/__postgres_database/gencode-remote | 6 +++--- cdist/conf/type/__postgres_role/gencode-remote | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 61cfa50d..9f12d215 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -41,12 +41,12 @@ if [ "$state_should" != "$state_is" ]; then present) owner="" if [ -f "$__object/parameter/owner" ]; then - owner="-O '$(cat "$__object/parameter/owner")'" + owner="-O \"$(cat "$__object/parameter/owner")\"" fi - echo "su - '$postgres_user' -c \"createdb $owner '$name'\"" + echo "su - '$postgres_user' -c \"createdb $owner \"$name\"\"" ;; absent) - echo "su - '$postgres_user' -c \"dropdb '$name'\"" + echo "su - '$postgres_user' -c \"dropdb \"$name\"\"" ;; esac fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index fd56e85d..0b8852f4 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -54,7 +54,7 @@ case "$state_should" in [ -n "$password" ] && password="PASSWORD '$password'" - cmd="CREATE ROLE $name WITH $password $booleans" + cmd="CREATE ROLE \"$name\" WITH $password $booleans" echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\"" ;; absent) From c32e4040b1b17f1e4c8173f23f1bdb2d5112f227 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 31 Dec 2019 19:16:49 +0200 Subject: [PATCH 036/138] __mysql_*: initial rewrite --- .../conf/type/__mysql_database/explorer/state | 15 ++++ .../conf/type/__mysql_database/gencode-remote | 72 ++++++------------- cdist/conf/type/__mysql_database/man.rst | 49 ------------- cdist/conf/type/__mysql_database/manifest | 26 +++++++ .../__mysql_database/parameter/default/state | 1 + .../type/__mysql_database/parameter/optional | 1 + .../type/__mysql_privileges/explorer/state | 22 ++++++ .../type/__mysql_privileges/gencode-remote | 31 ++++++++ .../__mysql_privileges/parameter/default/host | 1 + .../parameter/default/privileges | 1 + .../parameter/default/state | 1 + .../parameter/default/table | 1 + .../__mysql_privileges/parameter/optional | 4 ++ .../__mysql_privileges/parameter/required | 2 + cdist/conf/type/__mysql_user/explorer/state | 36 ++++++++++ cdist/conf/type/__mysql_user/gencode-remote | 50 +++++++++++++ .../type/__mysql_user/parameter/default/host | 1 + .../type/__mysql_user/parameter/default/state | 1 + .../conf/type/__mysql_user/parameter/optional | 4 ++ 19 files changed, 221 insertions(+), 98 deletions(-) create mode 100755 cdist/conf/type/__mysql_database/explorer/state delete mode 100644 cdist/conf/type/__mysql_database/man.rst create mode 100755 cdist/conf/type/__mysql_database/manifest create mode 100644 cdist/conf/type/__mysql_database/parameter/default/state create mode 100755 cdist/conf/type/__mysql_privileges/explorer/state create mode 100755 cdist/conf/type/__mysql_privileges/gencode-remote create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/host create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/privileges create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/state create mode 100644 cdist/conf/type/__mysql_privileges/parameter/default/table create mode 100644 cdist/conf/type/__mysql_privileges/parameter/optional create mode 100644 cdist/conf/type/__mysql_privileges/parameter/required create mode 100755 cdist/conf/type/__mysql_user/explorer/state create mode 100755 cdist/conf/type/__mysql_user/gencode-remote create mode 100644 cdist/conf/type/__mysql_user/parameter/default/host create mode 100644 cdist/conf/type/__mysql_user/parameter/default/state create mode 100644 cdist/conf/type/__mysql_user/parameter/optional diff --git a/cdist/conf/type/__mysql_database/explorer/state b/cdist/conf/type/__mysql_database/explorer/state new file mode 100755 index 00000000..16cc9ce5 --- /dev/null +++ b/cdist/conf/type/__mysql_database/explorer/state @@ -0,0 +1,15 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/name" ] +then + name="$( cat "$__object/parameter/name" )" +else + name="$__object_id" +fi + +if [ -n "$( mysql -B -N -e "show databases like '$name'" )" ] +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__mysql_database/gencode-remote b/cdist/conf/type/__mysql_database/gencode-remote index 23e51b05..d3692572 100755 --- a/cdist/conf/type/__mysql_database/gencode-remote +++ b/cdist/conf/type/__mysql_database/gencode-remote @@ -1,54 +1,28 @@ #!/bin/sh -e -# -# 2012 Benedikt Koeppel (code@benediktkoeppel.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# if --database was specified -if [ -f "$__object/parameter/name" ]; then - database="$(cat "$__object/parameter/name")" -else # otherwise use the object id as database name - database="$__object_id" +state_is="$( cat "$__object/explorer/state" )" + +state_should="$( cat "$__object/parameter/state" )" + +if [ "$state_is" = "$state_should" ] +then + exit 0 fi -cat <<-EOFF -mysql -u root <<-EOF - CREATE DATABASE IF NOT EXISTS $database -EOF -EOFF - -# if --user was specified -if [ -f "$__object/parameter/user" ]; then - user="$(cat "$__object/parameter/user")" - - # if --password was specified - if [ -f "$__object/parameter/password" ]; then - password="$(cat "$__object/parameter/password")" - cat <<-EOFF - mysql -u root <<-EOF - GRANT ALL PRIVILEGES ON $database.* to '$user'@'localhost' IDENTIFIED BY '$password'; -EOF -EOFF - else - cat <<-EOFF - mysql -u root <<-EOF - GRANT ALL PRIVILEGES ON $database.* to '$user'@'localhost'; -EOF -EOFF - fi +if [ -f "$__object/parameter/name" ] +then + name="$( cat "$__object/parameter/name" )" +else + name="$__object_id" fi + +case "$state_should" in + present) + echo "mysql -e 'create database \`$name\`'" + echo "create database $name" >> "$__messages_out" + ;; + absent) + echo "mysql -e 'drop database \`$name\`'" + echo "drop database $name" >> "$__messages_out" + ;; +esac diff --git a/cdist/conf/type/__mysql_database/man.rst b/cdist/conf/type/__mysql_database/man.rst deleted file mode 100644 index 1e245a08..00000000 --- a/cdist/conf/type/__mysql_database/man.rst +++ /dev/null @@ -1,49 +0,0 @@ -cdist-type__mysql_database(7) -============================= - -NAME ----- -cdist-type__mysql_database - Manage a MySQL database - - -DESCRIPTION ------------ -This cdist type allows you to install a MySQL database. - - -REQUIRED PARAMETERS -------------------- -None. - -OPTIONAL PARAMETERS -------------------- -name - The name of the database to install - defaults to the object id - -user - A user that should have access to the database - -password - The password for the user who manages the database - - -EXAMPLES --------- - -.. code-block:: sh - - __mysql_database "cdist" --name "cdist" --user "myuser" --password "mypwd" - - -AUTHORS -------- -Benedikt Koeppel - - -COPYING -------- -Copyright \(C) 2012 Benedikt Koeppel. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__mysql_database/manifest b/cdist/conf/type/__mysql_database/manifest new file mode 100755 index 00000000..a57c31ce --- /dev/null +++ b/cdist/conf/type/__mysql_database/manifest @@ -0,0 +1,26 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/user" ] +then + user="$( cat "$__object/parameter/user" )" +fi + +if [ -f "$__object/parameter/password" ] +then + password="$( cat "$__object/parameter/password" )" +fi + +if [ -n "$user" ] && [ -n "$password" ] +then + if [ -f "$__object/parameter/name" ] + then + database="$( cat "$__object/parameter/name" )" + else + database="$__object_id" + fi + + __mysql_user "$user" --password "$password" + + require="__mysql_user/$user" \ + __mysql_privileges "$database/$user" --database "$database" --user "$user" +fi diff --git a/cdist/conf/type/__mysql_database/parameter/default/state b/cdist/conf/type/__mysql_database/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__mysql_database/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__mysql_database/parameter/optional b/cdist/conf/type/__mysql_database/parameter/optional index 756afee7..6c0b1e85 100644 --- a/cdist/conf/type/__mysql_database/parameter/optional +++ b/cdist/conf/type/__mysql_database/parameter/optional @@ -1,3 +1,4 @@ name user password +state diff --git a/cdist/conf/type/__mysql_privileges/explorer/state b/cdist/conf/type/__mysql_privileges/explorer/state new file mode 100755 index 00000000..97674479 --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/explorer/state @@ -0,0 +1,22 @@ +#!/bin/sh -e + +privileges="$( cat "$__object/parameter/privileges" )" + +database="$( cat "$__object/parameter/database" )" + +table="$( cat "$__object/parameter/table" )" + +user="$( cat "$__object/parameter/user" )" + +host="$( cat "$__object/parameter/host" )" + +check_privileges="$( + mysql -B -N -e "show grants for '$user'@'$host'" \ + | grep -Ei "^grant $privileges on .$database.\..$table. to " || true )" + +if [ -n "$check_privileges" ] +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote new file mode 100755 index 00000000..6b2e0fc1 --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/gencode-remote @@ -0,0 +1,31 @@ +#!/bin/sh -e + +state_is="$( cat "$__object/explorer/state" )" + +state_should="$( cat "$__object/parameter/state" )" + +if [ "$state_is" = "$state_should" ] +then + exit 0 +fi + +privileges="$( cat "$__object/parameter/privileges" )" + +database="$( cat "$__object/parameter/database" )" + +table="$( cat "$__object/parameter/table" )" + +user="$( cat "$__object/parameter/user" )" + +host="$( cat "$__object/parameter/host" )" + +case "$state_should" in + present) + echo "mysql -e 'grant $privileges on \`$database\`.\`$table\` to \`$user\`@\`$host\`'" + echo "grant $privileges on $database.$table to $user@$host" >> "$__messages_out" + ;; + absent) + echo "mysql -e 'revoke $privileges on \`$database\`.\`$table\` from \`$user\`@\`$host\`'" + echo "revoke $privileges on $database.$table from $user@$host" >> "$__messages_out" + ;; +esac diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/host b/cdist/conf/type/__mysql_privileges/parameter/default/host new file mode 100644 index 00000000..2fbb50c4 --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/parameter/default/host @@ -0,0 +1 @@ +localhost diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/privileges b/cdist/conf/type/__mysql_privileges/parameter/default/privileges new file mode 100644 index 00000000..5472efad --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/parameter/default/privileges @@ -0,0 +1 @@ +all privileges diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/state b/cdist/conf/type/__mysql_privileges/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__mysql_privileges/parameter/default/table b/cdist/conf/type/__mysql_privileges/parameter/default/table new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/parameter/default/table @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__mysql_privileges/parameter/optional b/cdist/conf/type/__mysql_privileges/parameter/optional new file mode 100644 index 00000000..d4ed5bc5 --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/parameter/optional @@ -0,0 +1,4 @@ +privileges +table +host +state diff --git a/cdist/conf/type/__mysql_privileges/parameter/required b/cdist/conf/type/__mysql_privileges/parameter/required new file mode 100644 index 00000000..152b4a1e --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/parameter/required @@ -0,0 +1,2 @@ +database +user diff --git a/cdist/conf/type/__mysql_user/explorer/state b/cdist/conf/type/__mysql_user/explorer/state new file mode 100755 index 00000000..c91bb36a --- /dev/null +++ b/cdist/conf/type/__mysql_user/explorer/state @@ -0,0 +1,36 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/name" ] +then + name="$( cat "$__object/parameter/name" )" +else + name="$__object_id" +fi + +if [ -f "$__object/parameter/password" ] +then + password="$( cat "$__object/parameter/password" )" +else + password='' +fi + +host="$( cat "$__object/parameter/host" )" + +check_user="$( mysql -B -N -e "select user from mysql.user where user = '$name' and host = '$host'" )" + +if [ -n "$check_user" ] +then + if [ -n "$password" ] + then + check_password="$( mysql -B -N -e "select user from mysql.user where user = '$name' and host = '$host' and password = password( '$password' )" )" + fi + + if [ -n "$password" ] && [ -z "$check_password" ] + then + echo 'change-password' + else + echo 'present' + fi +else + echo 'absent' +fi diff --git a/cdist/conf/type/__mysql_user/gencode-remote b/cdist/conf/type/__mysql_user/gencode-remote new file mode 100755 index 00000000..67500716 --- /dev/null +++ b/cdist/conf/type/__mysql_user/gencode-remote @@ -0,0 +1,50 @@ +#!/bin/sh -e + +state_is="$( cat "$__object/explorer/state" )" + +state_should="$( cat "$__object/parameter/state" )" + +if [ "$state_is" = "$state_should" ] +then + exit 0 +fi + +if [ -f "$__object/parameter/name" ] +then + name="$( cat "$__object/parameter/name" )" +else + name="$__object_id" +fi + +host="$( cat "$__object/parameter/host" )" + +if [ -f "$__object/parameter/password" ] +then + password="$( cat "$__object/parameter/password" )" +else + if [ "$state_should" = 'present' ] + then + echo '--password needed' >&2 + exit 1 + else + password='' + fi +fi + +if [ "$state_is" = 'absent' ] && [ "$state_should" = 'present' ] +then + echo "mysql -e 'create user \`$name\`@\`$host\` identified by \"$password\"'" + echo "create user $name@$host" >> "$__messages_out" + +elif [ "$state_is" != 'absent' ] && [ "$state_should" = 'absent' ] +then + echo "mysql -e 'drop user \`$name\`@\`$host\`'" + echo "drop user $name@$host" >> "$__messages_out" + +elif [ "$state_is" = 'change-password' ] +then + # this only works with MySQL 5.7.6 and later or MariaDB 10.1.20 and later + echo "mysql -e 'alter user \`$name\`@\`$host\` identified by \"$password\"'" + echo "mysql -e 'flush privileges'" + echo "change password $name@$host" >> "$__messages_out" +fi diff --git a/cdist/conf/type/__mysql_user/parameter/default/host b/cdist/conf/type/__mysql_user/parameter/default/host new file mode 100644 index 00000000..2fbb50c4 --- /dev/null +++ b/cdist/conf/type/__mysql_user/parameter/default/host @@ -0,0 +1 @@ +localhost diff --git a/cdist/conf/type/__mysql_user/parameter/default/state b/cdist/conf/type/__mysql_user/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__mysql_user/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__mysql_user/parameter/optional b/cdist/conf/type/__mysql_user/parameter/optional new file mode 100644 index 00000000..a286266c --- /dev/null +++ b/cdist/conf/type/__mysql_user/parameter/optional @@ -0,0 +1,4 @@ +name +host +password +state From 9a693537f4192e3cd133e14cd31ab1bdcc792608 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 1 Jan 2020 12:38:12 +0200 Subject: [PATCH 037/138] __mysql_*: add license headers --- .../conf/type/__mysql_database/explorer/state | 18 ++++++++++++++++++ .../conf/type/__mysql_database/gencode-remote | 18 ++++++++++++++++++ cdist/conf/type/__mysql_database/manifest | 18 ++++++++++++++++++ .../type/__mysql_privileges/explorer/state | 18 ++++++++++++++++++ .../type/__mysql_privileges/gencode-remote | 18 ++++++++++++++++++ cdist/conf/type/__mysql_user/explorer/state | 18 ++++++++++++++++++ cdist/conf/type/__mysql_user/gencode-remote | 18 ++++++++++++++++++ 7 files changed, 126 insertions(+) diff --git a/cdist/conf/type/__mysql_database/explorer/state b/cdist/conf/type/__mysql_database/explorer/state index 16cc9ce5..79858695 100755 --- a/cdist/conf/type/__mysql_database/explorer/state +++ b/cdist/conf/type/__mysql_database/explorer/state @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# if [ -f "$__object/parameter/name" ] then diff --git a/cdist/conf/type/__mysql_database/gencode-remote b/cdist/conf/type/__mysql_database/gencode-remote index d3692572..1bdb2b11 100755 --- a/cdist/conf/type/__mysql_database/gencode-remote +++ b/cdist/conf/type/__mysql_database/gencode-remote @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# state_is="$( cat "$__object/explorer/state" )" diff --git a/cdist/conf/type/__mysql_database/manifest b/cdist/conf/type/__mysql_database/manifest index a57c31ce..628b543c 100755 --- a/cdist/conf/type/__mysql_database/manifest +++ b/cdist/conf/type/__mysql_database/manifest @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# if [ -f "$__object/parameter/user" ] then diff --git a/cdist/conf/type/__mysql_privileges/explorer/state b/cdist/conf/type/__mysql_privileges/explorer/state index 97674479..0cfbaacd 100755 --- a/cdist/conf/type/__mysql_privileges/explorer/state +++ b/cdist/conf/type/__mysql_privileges/explorer/state @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# privileges="$( cat "$__object/parameter/privileges" )" diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote index 6b2e0fc1..bcd362e6 100755 --- a/cdist/conf/type/__mysql_privileges/gencode-remote +++ b/cdist/conf/type/__mysql_privileges/gencode-remote @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# state_is="$( cat "$__object/explorer/state" )" diff --git a/cdist/conf/type/__mysql_user/explorer/state b/cdist/conf/type/__mysql_user/explorer/state index c91bb36a..6817ee9d 100755 --- a/cdist/conf/type/__mysql_user/explorer/state +++ b/cdist/conf/type/__mysql_user/explorer/state @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# if [ -f "$__object/parameter/name" ] then diff --git a/cdist/conf/type/__mysql_user/gencode-remote b/cdist/conf/type/__mysql_user/gencode-remote index 67500716..5f13bc87 100755 --- a/cdist/conf/type/__mysql_user/gencode-remote +++ b/cdist/conf/type/__mysql_user/gencode-remote @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# state_is="$( cat "$__object/explorer/state" )" From 24862e0208705cd9081f5963d1f07e0fcceb23f2 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 3 Jan 2020 18:26:11 +0200 Subject: [PATCH 038/138] __mysql_database: carry over state --- cdist/conf/type/__mysql_database/manifest | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__mysql_database/manifest b/cdist/conf/type/__mysql_database/manifest index 628b543c..a3c9ed5d 100755 --- a/cdist/conf/type/__mysql_database/manifest +++ b/cdist/conf/type/__mysql_database/manifest @@ -37,8 +37,16 @@ then database="$__object_id" fi - __mysql_user "$user" --password "$password" + state_should="$( cat "$__object/parameter/state" )" + __mysql_user "$user" \ + --password "$password" \ + --state "$state_should" + + # removing user should remove all user's privileges require="__mysql_user/$user" \ - __mysql_privileges "$database/$user" --database "$database" --user "$user" + __mysql_privileges "$database/$user" \ + --database "$database" \ + --user "$user" \ + --state "$state_should" fi From fcc774cb7b2b3f2128dc77d622d7801397d906b7 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 3 Jan 2020 18:33:23 +0200 Subject: [PATCH 039/138] __mysql_database: add manual --- cdist/conf/type/__mysql_database/man.rst | 55 ++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 cdist/conf/type/__mysql_database/man.rst diff --git a/cdist/conf/type/__mysql_database/man.rst b/cdist/conf/type/__mysql_database/man.rst new file mode 100644 index 00000000..b3b56b5f --- /dev/null +++ b/cdist/conf/type/__mysql_database/man.rst @@ -0,0 +1,55 @@ +cdist-type__mysql_database(7) +============================= + +NAME +---- +cdist-type__mysql_database - Manage a MySQL database + + +DESCRIPTION +----------- + +Create MySQL database and optionally user with all privileges. + + +OPTIONAL PARAMETERS +------------------- +name + Name of database. Defaults to object id. + +user + Create user and give all privileges to database. + +password + Password for user. + +state + Defaults to present. + If absent and user is also set, both will be removed (with privileges). + + +EXAMPLES +-------- + +.. code-block:: sh + + # just create database + __mysql_database foo + + # create database with respective user with all privileges to database + __mysql_database bar \ + --user name \ + --password secret + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version. From 5e8dc7122d764896247258ae4c9049e222f4d7ff Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 3 Jan 2020 18:48:11 +0200 Subject: [PATCH 040/138] __mysql_user: add manual --- cdist/conf/type/__mysql_user/man.rst | 48 ++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 cdist/conf/type/__mysql_user/man.rst diff --git a/cdist/conf/type/__mysql_user/man.rst b/cdist/conf/type/__mysql_user/man.rst new file mode 100644 index 00000000..c2b222d5 --- /dev/null +++ b/cdist/conf/type/__mysql_user/man.rst @@ -0,0 +1,48 @@ +cdist-type__mysql_user(7) +========================= + +NAME +---- +cdist-type__mysql_user - Manage a MySQL user + + +DESCRIPTION +----------- + +Create MySQL user or change password for the user. + + +OPTIONAL PARAMETERS +------------------- +name + Name of user. Defaults to object id. + +host + Host of user. Defaults to localhost. + +password + Password of user. + +state + Defaults to present. + + +EXAMPLES +-------- + +.. code-block:: sh + + __mysql_user user --password secret + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version. From 4329cced82930a336378765b294b9a3be9433991 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 3 Jan 2020 18:55:55 +0200 Subject: [PATCH 041/138] __mysql_privileges: add manual --- cdist/conf/type/__mysql_privileges/man.rst | 57 ++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 cdist/conf/type/__mysql_privileges/man.rst diff --git a/cdist/conf/type/__mysql_privileges/man.rst b/cdist/conf/type/__mysql_privileges/man.rst new file mode 100644 index 00000000..8208d7d4 --- /dev/null +++ b/cdist/conf/type/__mysql_privileges/man.rst @@ -0,0 +1,57 @@ +cdist-type__mysql_privileges(7) +=============================== + +NAME +---- +cdist-type__mysql_privileges - Manage MySQL privileges + + +DESCRIPTION +----------- + +Grant and revoke privileges of MySQL user. + + +REQUIRED PARAMETERS +------------------- +database + Name of database. + +User + Name of user. + + +OPTIONAL PARAMETERS +------------------- +privileges + Defaults to "all". + +table + Defaults to "*". + +host + Defaults to localhost. + +state + "present" grants and "absent" revokes. Defaults to present. + + +EXAMPLES +-------- + +.. code-block:: sh + + __mysql_privileges user-to-db --database db --user user + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version. From 6369bc1ae5e17dd9c0a13d57419107cbdb166245 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 10:07:53 +0100 Subject: [PATCH 042/138] ++changelog --- docs/changelog | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/changelog b/docs/changelog index 415af0af..8feb0b83 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,8 +2,9 @@ Changelog --------- next: - * Type __consul_agent: don't deploy init script on Alpine anymore, - it ships with one itself (Nico Schottelius) + * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) + * Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong) + * Types __postgres_*: Use double quoted identifiers in generated SQL (fnux) 6.3.0: 2019-12-12 * Type __package_update_index: Fix Alpine part (Dominique Roux) From 72935e0a797461d39f64b9f85bac02ad06347b1d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 11:08:48 +0100 Subject: [PATCH 043/138] ++changelog --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8feb0b83..d0a3a2de 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,7 +4,7 @@ Changelog next: * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) * Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong) - * Types __postgres_*: Use double quoted identifiers in generated SQL (fnux) + * Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure) 6.3.0: 2019-12-12 * Type __package_update_index: Fix Alpine part (Dominique Roux) From e4596593c08f43284e1fa85d1e353b733bebdc57 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 30 Dec 2019 09:32:15 +0100 Subject: [PATCH 044/138] Add cdist info command --- cdist/argparse.py | 32 +++++++ cdist/exec/local.py | 42 ++------- cdist/exec/util.py | 25 ++++++ cdist/info.py | 183 ++++++++++++++++++++++++++++++++++++++++ docs/src/man1/cdist.rst | 37 +++++++- 5 files changed, 283 insertions(+), 36 deletions(-) create mode 100644 cdist/info.py diff --git a/cdist/argparse.py b/cdist/argparse.py index 7dc683f3..611c484a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -6,6 +6,7 @@ import collections import functools import cdist.configuration import cdist.preos +import cdist.info # set of beta sub-commands @@ -436,6 +437,37 @@ def get_parsers(): ' should be POSIX compatible shell.')) parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) + # Info + parser['info'] = parser['sub'].add_parser('info') + parser['info'].add_argument( + '-a', '--all', help='Display all info. This is the default.', + action='store_true', default=False) + parser['info'].add_argument( + '-c', '--conf-dir', + help='Add configuration directory (can be repeated).', + action='append') + parser['info'].add_argument( + '-e', '--global-explorers', + help='Display info for global explorers.', action='store_true', + default=False) + parser['info'].add_argument( + '-F', '--fixed-string', + help='Interpret pattern as a fixed string.', action='store_true', + default=False) + parser['info'].add_argument( + '-f', '--full', help='Display full details.', + action='store_true', default=False) + parser['info'].add_argument( + '-g', '--config-file', + help='Use specified custom configuration file.', + dest="config_file", required=False) + parser['info'].add_argument( + '-t', '--types', help='Display info for types.', + action='store_true', default=False) + parser['info'].add_argument( + 'pattern', nargs='?', help='Glob pattern.') + parser['info'].set_defaults(func=cdist.info.Info.commandline) + for p in parser: parser[p].epilog = EPILOG diff --git a/cdist/exec/local.py b/cdist/exec/local.py index f83c85df..ad6c6e36 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -69,7 +69,6 @@ class Local(object): self.exec_path = exec_path self.custom_initial_manifest = initial_manifest - self._add_conf_dirs = add_conf_dirs self.cache_path_pattern = cache_path_pattern self.quiet_mode = quiet_mode if configuration: @@ -84,16 +83,7 @@ class Local(object): self._init_cache_dir(None) self._init_paths() self._init_object_marker() - self._init_conf_dirs() - - @property - def dist_conf_dir(self): - return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), - "conf")) - - @property - def home_dir(self): - return cdist.home_dir() + self._init_conf_dirs(add_conf_dirs) def _init_log(self): self.log = logging.getLogger(self.target_host[0]) @@ -140,28 +130,9 @@ class Local(object): # Does not need to be secure - just randomly different from .cdist self.object_marker_name = tempfile.mktemp(prefix='.cdist-', dir='') - def _init_conf_dirs(self): - self.conf_dirs = [] - - self.conf_dirs.append(self.dist_conf_dir) - - # Is the default place for user created explorer, type and manifest - if self.home_dir: - self.conf_dirs.append(self.home_dir) - - # Add directories defined in the CDIST_PATH environment variable - # if 'CDIST_PATH' in os.environ: - # cdist_path_dirs = re.split(r'(?. +# +# + +import cdist +import cdist.configuration +import cdist.core +import cdist.exec.util as util +import os +import glob +import fnmatch + + +class Info(object): + + def __init__(self, conf_dirs, args): + self.conf_dirs = conf_dirs + self.all = args.all + self.display_global_explorers = args.global_explorers + self.display_types = args.types + if not self.display_global_explorers and not self.display_types: + self.all = True + self.fixed_string = args.fixed_string + self._setup_glob_pattern(args.pattern) + self.full = args.full + + def _setup_glob_pattern(self, pattern): + if pattern is None: + self.glob_pattern = '*' + elif ('?' in pattern or '*' in pattern or '[' in pattern or + self.fixed_string): + self.glob_pattern = pattern + else: + self.glob_pattern = '*' + pattern + '*' + + @classmethod + def commandline(cls, args): + cfg = cdist.configuration.Configuration(args) + configuration = cfg.get_config(section='GLOBAL') + conf_dirs = util.resolve_conf_dirs(configuration, + args.conf_dir) + c = cls(conf_dirs, args) + c.run() + + def _get_global_explorers(self, conf_path): + rv = [] + global_explorer_path = os.path.join(conf_path, "explorer", + self.glob_pattern) + if self.fixed_string: + if os.path.exists(global_explorer_path): + rv.append(global_explorer_path) + else: + for explorer in glob.glob(global_explorer_path): + rv.append(explorer) + return rv + + def _should_display_type(self, dir_entry): + if not dir_entry.is_dir(): + return False + if self.glob_pattern is None: + return True + if self.fixed_string: + return dir_entry.name == self.glob_pattern + else: + return fnmatch.fnmatch(dir_entry.name, self.glob_pattern) + + def _get_types(self, conf_path): + rv = [] + types_path = os.path.join(conf_path, "type") + if not os.path.exists(types_path): + return rv + with os.scandir(types_path) as it: + for entry in it: + if self._should_display_type(entry): + rv.append(entry.path) + return rv + + def _display_details(self, title, details, default_values=None, + deprecated=None): + if not details: + return + if isinstance(details, bool): + print("\t{}: {}".format(title, 'yes' if details else 'no')) + elif isinstance(details, str): + print("\t{}: {}".format(title, details)) + elif isinstance(details, list): + dv = dict(default_values) if default_values else {} + dp = dict(deprecated) if deprecated else {} + + print("\t{}:".format(title)) + for x in sorted(details): + print("\t\t{}".format(x), end='') + has_default = x in dv + is_deprecated = x in dp + need_comma = False + if has_default or is_deprecated: + print(" (", end='') + if has_default: + print("default: {}".format(dv[x]), end='') + need_comma = True + if is_deprecated: + print("{}deprecated".format(', ' if need_comma else ''), + end='') + if has_default or is_deprecated: + print(")", end='') + print() + + def _display_type_parameters(self, cdist_type): + self._display_details("required parameters", + cdist_type.required_parameters, + default_values=cdist_type.parameter_defaults, + deprecated=cdist_type.deprecated_parameters) + self._display_details("required multiple parameters", + cdist_type.required_multiple_parameters, + default_values=cdist_type.parameter_defaults, + deprecated=cdist_type.deprecated_parameters) + self._display_details("optional parameters", + cdist_type.optional_parameters, + default_values=cdist_type.parameter_defaults, + deprecated=cdist_type.deprecated_parameters) + self._display_details("optional multiple parameters", + cdist_type.optional_multiple_parameters, + default_values=cdist_type.parameter_defaults, + deprecated=cdist_type.deprecated_parameters) + self._display_details("boolean parameters", + cdist_type.boolean_parameters, + default_values=cdist_type.parameter_defaults, + deprecated=cdist_type.deprecated_parameters) + + def _display_type_characteristics(self, cdist_type): + characteristics = [] + if cdist_type.is_install: + characteristics.append('install') + else: + characteristics.append('config') + if cdist_type.is_singleton: + characteristics.append('singleton') + if cdist_type.is_nonparallel: + characteristics.append('nonparallel') + else: + characteristics.append('parallel') + if cdist_type.deprecated is not None: + characteristics.append('deprecated') + print("\t{}".format(', '.join(characteristics))) + + def _display_type_details(self, type_path): + dirname, basename = os.path.split(type_path) + cdist_type = cdist.core.CdistType(dirname, basename) + + self._display_type_characteristics(cdist_type) + self._display_type_parameters(cdist_type) + + def run(self): + rv = [] + for conf_path in self.conf_dirs: + if self.all or self.display_global_explorers: + rv.extend((x, 'E', ) for x in self._get_global_explorers( + conf_path)) + if self.all or self.display_types: + rv.extend((x, 'T', ) for x in self._get_types(conf_path)) + rv = sorted(rv, key=lambda x: x[0]) + for x, t in rv: + print(x) + if self.full and t == 'T': + self._display_type_details(x) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 55db82ed..66c356ec 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,7 +11,7 @@ SYNOPSIS :: - cdist [-h] [-V] {banner,config,install,inventory,preos,shell} ... + cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info} ... cdist banner [-h] [-l LOGLEVEL] [-q] [-v] @@ -84,6 +84,8 @@ SYNOPSIS cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] + cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-t] [pattern] + DESCRIPTION ----------- @@ -604,6 +606,39 @@ usage. Its primary use is for debugging type parameters. be POSIX compatible shell. +INFO +---- +Display information for cdist (global explorers, types). + +**pattern** + Glob pattern. If it contains special characters('?', '*', '[') then it is + used as specified, otherwise it is translated to `*pattern*`. + +**-h, --help** + Show help message and exit. + +**-a, --all** + Display all info. This is the default. + +**-c CONF_DIR, --conf-dir CONF_DIR** + Add configuration directory (can be repeated). + +**-e, --global-explorers** + Display info for global explorers. + +**-F, --fixed-string** + Interpret pattern as a fixed string. + +**-f, --full** + Display full details. + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-t, --types** + Display info for types. + + CONFIGURATION ------------- cdist obtains configuration data from the following sources in the following From 7b1192257d8517cc838f84a8c57ac0964d104588 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 13:17:54 +0100 Subject: [PATCH 045/138] Fix incomplete cdist info synopsis --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 66c356ec..bc73a0b8 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -84,7 +84,7 @@ SYNOPSIS cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] - cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-t] [pattern] + cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] DESCRIPTION From e2015367925c8a2716e6a79f6f2609f2877cd134 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 13:18:13 +0100 Subject: [PATCH 046/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index d0a3a2de..7489489e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) * Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong) * Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure) + * Core: Add cdist info command (Darko Poljak) 6.3.0: 2019-12-12 * Type __package_update_index: Fix Alpine part (Dominique Roux) From bc1990c7c8244b2d00a91bc13dbf796d91c21041 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 13:44:04 +0100 Subject: [PATCH 047/138] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 7489489e..fc4d69a8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,8 @@ next: * Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong) * Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure) * Core: Add cdist info command (Darko Poljak) + * New types: __mysql_user, __mysql_privileges (Ander Punnar) + * Type __mysql_database: Rewrite (Ander Punnar) 6.3.0: 2019-12-12 * Type __package_update_index: Fix Alpine part (Dominique Roux) From 7c9dd3b03e31f39cbe758510e2aa1f542eae4825 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 14:36:14 +0100 Subject: [PATCH 048/138] Release 6.4.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index fc4d69a8..706d76af 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) * Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong) * Types __postgres_*: Use double quoted identifiers in generated SQL (Timothée Floure) From d1a64596fe73697d3e6a8e514991ffc173d04772 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 14:56:05 +0100 Subject: [PATCH 049/138] Update build-helper --- bin/build-helper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/build-helper b/bin/build-helper index 69dee4c7..ed41e438 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -370,7 +370,7 @@ eof cat << eof Manual steps post release: - cdist-web - - send mail body generated in mailinglist.tmp and inform Dmitry for deb + - send generated mailinglist.tmp mail - twitter eof ;; From 94e32dcd78d73ea5f09845cfdd4f488095f17145 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sat, 4 Jan 2020 16:36:44 +0200 Subject: [PATCH 050/138] __apt_unattended_upgrades: initial commit --- .../type/__apt_unattended_upgrades/manifest | 74 +++++++++++++++++++ .../parameter/boolean | 1 + .../parameter/optional | 1 + .../parameter/optional_multiple | 1 + .../type/__apt_unattended_upgrades/singleton | 0 5 files changed, 77 insertions(+) create mode 100755 cdist/conf/type/__apt_unattended_upgrades/manifest create mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/boolean create mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/optional create mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple create mode 100644 cdist/conf/type/__apt_unattended_upgrades/singleton diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest new file mode 100755 index 00000000..bf4d825a --- /dev/null +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -0,0 +1,74 @@ +#!/bin/sh -e + +__package unattended-upgrades + +# in normal circumstances 20auto-upgrades is managed +# by debconf and it can only contain these lines +# https://wiki.debian.org/UnattendedUpgrades + +require='__package/unattended-upgrades' \ + __file /etc/apt/apt.conf.d/20auto-upgrades \ + --owner root \ + --group root \ + --mode 644 \ + --source - << EOF +APT::Periodic::Update-Package-Lists "1"; +APT::Periodic::Unattended-Upgrade "1"; +EOF + +# lets not write into upstream 50unattended-upgrades file, +# but use our own config files to avoid clashes + +ml_conf='/etc/apt/apt.conf.d/51unattended-upgrades-mail' +bl_conf='/etc/apt/apt.conf.d/51unattended-upgrades-blacklist' + +if [ -f "$__object/parameter/mail" ] +then + mail="$( cat "$__object/parameter/mail" )" +else + mail='' +fi + +if [ -n "$mail" ] +then + if [ -f "$__object/parameter/mail-on-error" ] + then + mail_on_error='true' + else + mail_on_error='false' + fi + + __file "$ml_conf" \ + --owner root \ + --group root \ + --mode 644 \ + --source - << EOF +Unattended-Upgrade::Mail "$mail"; +Unattended-Upgrade::MailOnlyOnError "$mail_on_error"; +EOF + +else + __file "$ml_conf" --state absent +fi + +if [ -f "$__object/parameter/blacklist" ] +then + bl='Unattended-Upgrade::Package-Blacklist {'; + + while read -r l + do + bl="$( printf '%s\n"%s";\n' "$bl" "$l" )" + done \ + < "$__object/parameter/blacklist" + + bl="$( printf '%s\n}' "$bl" )" + + echo "$bl" \ + | __file "$bl_conf" \ + --owner root \ + --group root \ + --mode 644 \ + --source - +else + __file "$bl_conf" --state absent +fi diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean new file mode 100644 index 00000000..edcaa12a --- /dev/null +++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean @@ -0,0 +1 @@ +mail-on-error diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional new file mode 100644 index 00000000..fa7963cc --- /dev/null +++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional @@ -0,0 +1 @@ +mail diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple new file mode 100644 index 00000000..27b9ffc9 --- /dev/null +++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple @@ -0,0 +1 @@ +blacklist diff --git a/cdist/conf/type/__apt_unattended_upgrades/singleton b/cdist/conf/type/__apt_unattended_upgrades/singleton new file mode 100644 index 00000000..e69de29b From a9d491f998c2b41ad97bfeace7f1d24a73ff7482 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sat, 4 Jan 2020 16:59:42 +0200 Subject: [PATCH 051/138] __apt_unattended_upgrades: add license header --- .../type/__apt_unattended_upgrades/manifest | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index bf4d825a..88a5ccd8 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -1,4 +1,22 @@ #!/bin/sh -e +# +# 2020 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# __package unattended-upgrades From ec8d9571f971c0a0616acdccc9a32589f1b6f042 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sat, 4 Jan 2020 17:07:46 +0200 Subject: [PATCH 052/138] __apt_unattended_upgrades: add manual --- .../type/__apt_unattended_upgrades/man.rst | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 cdist/conf/type/__apt_unattended_upgrades/man.rst diff --git a/cdist/conf/type/__apt_unattended_upgrades/man.rst b/cdist/conf/type/__apt_unattended_upgrades/man.rst new file mode 100644 index 00000000..d64b2e9e --- /dev/null +++ b/cdist/conf/type/__apt_unattended_upgrades/man.rst @@ -0,0 +1,54 @@ +cdist-type__apt_unattended_upgrades(7) +====================================== + +NAME +---- +cdist-type__apt_unattended_upgrades - automatic installation of updates + + +DESCRIPTION +----------- + +Install and configure unattended-upgrades package. + + +OPTIONAL PARAMETERS +------------------- +mail + Send email to this address for problems or packages upgrades. + + +OPTIONAL MULTIPLE PARAMETERS +---------------------------- +blacklist + Python regular expressions, matching packages to exclude from upgrading. + + +BOOLEAN PARAMETERS +------------------ +mail-on-error + Get emails only on errors. + + +EXAMPLES +-------- + +.. code-block:: sh + + __apt_unattended_upgrades \ + --mail root \ + --mail-on-error \ + --blacklist multipath-tools \ + --blacklist open-iscsi + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) any +later version. From d4bd49bbb598dfd4e4a510a2bf9035dcb4686e4a Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sat, 4 Jan 2020 17:43:57 +0200 Subject: [PATCH 053/138] __acl: rename --acl to --entry for the sake of consistency, add compatibility --- cdist/conf/type/__acl/gencode-remote | 5 +++- cdist/conf/type/__acl/man.rst | 28 +++++++++---------- .../conf/type/__acl/parameter/deprecated/acl | 1 + .../type/__acl/parameter/optional_multiple | 1 + 4 files changed, 20 insertions(+), 15 deletions(-) create mode 100644 cdist/conf/type/__acl/parameter/deprecated/acl diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index 6dab4d09..f4f0d1e2 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -28,7 +28,10 @@ acl_path="/$__object_id" acl_is="$( cat "$__object/explorer/acl_is" )" -if [ -f "$__object/parameter/acl" ] +if [ -f "$__object/parameter/entry" ] +then + acl_should="$( cat "$__object/parameter/entry" )" +elif [ -f "$__object/parameter/acl" ] then acl_should="$( cat "$__object/parameter/acl" )" elif diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index 85e946ce..c3493e49 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -15,7 +15,7 @@ See ``setfacl`` and ``acl`` manpages for more details. REQUIRED MULTIPLE PARAMETERS ---------------------------- -acl +entry Set ACL entry following ``getfacl`` output syntax. @@ -36,8 +36,8 @@ remove DEPRECATED PARAMETERS --------------------- -Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they -will be removed in future versions. Please use ``acl`` parameter instead. +Parameters ``acl``, ``user``, ``group``, ``mask`` and ``other`` are deprecated and they +will be removed in future versions. Please use ``entry`` parameter instead. EXAMPLES @@ -49,27 +49,27 @@ EXAMPLES --default \ --recursive \ --remove \ - --acl user:alice:rwx \ - --acl user:bob:r-x \ - --acl group:project-group:rwx \ - --acl group:some-other-group:r-x \ - --acl mask::r-x \ - --acl other::r-x + --entry user:alice:rwx \ + --entry user:bob:r-x \ + --entry group:project-group:rwx \ + --entry group:some-other-group:r-x \ + --entry mask::r-x \ + --entry other::r-x # give Alice read-only access to subdir, # but don't allow her to see parent content. __acl /srv/project2 \ --remove \ - --acl default:group:secret-project:rwx \ - --acl group:secret-project:rwx \ - --acl user:alice:--x + --entry default:group:secret-project:rwx \ + --entry group:secret-project:rwx \ + --entry user:alice:--x __acl /srv/project2/subdir \ --default \ --remove \ - --acl group:secret-project:rwx \ - --acl user:alice:r-x + --entry group:secret-project:rwx \ + --entry user:alice:r-x AUTHORS diff --git a/cdist/conf/type/__acl/parameter/deprecated/acl b/cdist/conf/type/__acl/parameter/deprecated/acl new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/acl @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple index 95c25d55..c615d507 100644 --- a/cdist/conf/type/__acl/parameter/optional_multiple +++ b/cdist/conf/type/__acl/parameter/optional_multiple @@ -1,3 +1,4 @@ +entry acl user group From 51ba4a49d8ec79968f79563f994489c619f10bac Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 4 Jan 2020 18:21:23 +0100 Subject: [PATCH 054/138] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 706d76af..a7bcf9b1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) + 6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) * Type __install_chroot_umount: Bugfix: type was not using __chroot_umount/manifest (Steven Armstrong) From 11f569959d6e331d4d5052ca73fb5d83bf9df8e7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 11 Jan 2020 14:16:33 +0100 Subject: [PATCH 055/138] Fix missing configuration file usage, support -g PreOS code did not use configuration support. This fix adds support for using cdist configuration, which takes into account cdist configuration file, environment variables and command line options, especially conf_dir. It also adds support for -g, --config-file option, for specifying custom configuration file. --- cdist/preos.py | 24 +++++++++++------------- docs/changelog | 1 + docs/src/man1/cdist.rst | 7 ++++++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 378071db..491338d2 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -5,8 +5,9 @@ import inspect import argparse import cdist import logging -import re import cdist.argparse +import cdist.configuration +import cdist.exec.util as util _PREOS_CALL = "commandline" @@ -24,16 +25,6 @@ def extend_plugins_path(dirs): _PLUGINS_PATH.append(preos_dir) -cdist_home = cdist.home_dir() -if cdist_home: - extend_plugins_path((cdist_home, )) -x = 'CDIST_PATH' -if x in os.environ: - vals = re.split(r'(? Date: Sat, 11 Jan 2020 15:26:46 +0100 Subject: [PATCH 056/138] Info command: support tilde expansion --- cdist/exec/util.py | 7 +++++++ cdist/info.py | 8 +++----- cdist/preos.py | 7 ++----- docs/changelog | 1 + 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 5513f01d..9787f431 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -24,6 +24,7 @@ import os from tempfile import TemporaryFile import cdist +import cdist.configuration # IMPORTANT: @@ -200,3 +201,9 @@ def resolve_conf_dirs(configuration, add_conf_dirs): conf_dirs.extend(add_conf_dirs) conf_dirs = set(conf_dirs) return conf_dirs + + +def resolve_conf_dirs_from_config_and_args(args): + cfg = cdist.configuration.Configuration(args) + configuration = cfg.get_config(section='GLOBAL') + return resolve_conf_dirs(configuration, args.conf_dir) diff --git a/cdist/info.py b/cdist/info.py index 4c1d3560..b896a3d1 100644 --- a/cdist/info.py +++ b/cdist/info.py @@ -53,10 +53,7 @@ class Info(object): @classmethod def commandline(cls, args): - cfg = cdist.configuration.Configuration(args) - configuration = cfg.get_config(section='GLOBAL') - conf_dirs = util.resolve_conf_dirs(configuration, - args.conf_dir) + conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) c = cls(conf_dirs, args) c.run() @@ -170,7 +167,8 @@ class Info(object): def run(self): rv = [] - for conf_path in self.conf_dirs: + for cp in self.conf_dirs: + conf_path = os.path.expanduser(cp) if self.all or self.display_global_explorers: rv.extend((x, 'E', ) for x in self._get_global_explorers( conf_path)) diff --git a/cdist/preos.py b/cdist/preos.py index 491338d2..e353fe3b 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -101,13 +101,10 @@ class PreOS(object): action='store_true', default=False) parser.add_argument('remainder_args', nargs=argparse.REMAINDER) args = parser.parse_args(argv[1:]) - cdist.argparse.handle_loglevel(args) + st.argparse.handle_loglevel(args) log.debug("preos args : {}".format(args)) - cfg = cdist.configuration.Configuration(args) - configuration = cfg.get_config(section='GLOBAL') - conf_dirs = util.resolve_conf_dirs(configuration, - args.conf_dir) + conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) extend_plugins_path(conf_dirs) sys.path.extend(_PLUGINS_PATH) diff --git a/docs/changelog b/docs/changelog index 1b1a909e..526fc320 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) * Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak) + * Core info command: Support tilde expansion of conf directories (Darko Poljak) 6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) From 3258fc98e15fedbd98e17f7d0b568a38b8da139c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 12 Jan 2020 12:19:49 +0100 Subject: [PATCH 057/138] Fix typo --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index e353fe3b..bf2a8e60 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -101,7 +101,7 @@ class PreOS(object): action='store_true', default=False) parser.add_argument('remainder_args', nargs=argparse.REMAINDER) args = parser.parse_args(argv[1:]) - st.argparse.handle_loglevel(args) + cdist.argparse.handle_loglevel(args) log.debug("preos args : {}".format(args)) conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) From 93ec4b46aab9cd61d66cb375fcf7a2599c5ef9bb Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 15 Jan 2020 17:23:13 +0100 Subject: [PATCH 058/138] [__line] Ensure the line is only added once --- cdist/conf/type/__line/explorer/state | 22 +++++++++++----------- cdist/conf/type/__line/gencode-remote | 3 +++ 2 files changed, 14 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index 2ef252c8..9c0dd1b2 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -18,6 +18,17 @@ # along with cdist. If not, see . # +if [ -f "$__object/parameter/file" ]; then + file="$(cat "$__object/parameter/file")" +else + file="/$__object_id" +fi + +if [ ! -f "$file" ]; then + echo "file_missing" + exit 0 +fi + if [ -f "$__object/parameter/before" ]; then position="before" elif [ -f "$__object/parameter/after" ]; then @@ -33,17 +44,6 @@ else needle="line" fi -if [ -f "$__object/parameter/file" ]; then - file="$(cat "$__object/parameter/file")" -else - file="/$__object_id" -fi - -if [ ! -f "$file" ]; then - echo "file_missing" - exit 0 -fi - awk -v position="$position" -v needle="$needle" ' function _find(_text, _pattern) { if (needle == "regex") { diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 03e90c1b..0dd8609a 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2018 Steven Armstrong (steven-cdist at armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -104,10 +105,12 @@ BEGIN { if (anchor && match(\$0, anchor)) { if (position == "before") { print line + add = 0 print } else if (position == "after") { print print line + add = 0 } next } From 629d0795c80bdf8af83a71712c643275be0799f1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 15 Jan 2020 17:23:26 +0100 Subject: [PATCH 059/138] [__line] Always add line to end if anchor is not found --- cdist/conf/type/__line/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 0dd8609a..c8c90c38 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -118,7 +118,7 @@ BEGIN { print } END { - if (add && position == "end") { + if (add) { print line } } From 4cdb8aaa03d07aa72de8dd5961844699ad5888bd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 15 Jan 2020 17:39:21 +0100 Subject: [PATCH 060/138] [__line/state] Make sure the index match is at the beginning Without the == 1 all lines which contain --line as a substring match. e.g. if --line is "line" and the file contains the line "wrong line" this was considered a match. --- cdist/conf/type/__line/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index 9c0dd1b2..28ec35e2 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -49,7 +49,7 @@ function _find(_text, _pattern) { if (needle == "regex") { return match(_text, _pattern) } else { - return index(_text, _pattern) + return index(_text, _pattern) == 1 } } BEGIN { From 51b1b11cc21e257acbce420ecfcd48ec37e66705 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 15 Jan 2020 17:54:40 +0100 Subject: [PATCH 061/138] [__line/state] Logic fixes in explorer This commit fixes the incorrectly reported state "wrongposition" if position is "after" and anchor is present in the file but the line missing. --- cdist/conf/type/__line/explorer/state | 41 +++++++++++++++------------ 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index 28ec35e2..6ff0a798 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2018 Steven Armstrong (steven-cdist at armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,7 +20,7 @@ # if [ -f "$__object/parameter/file" ]; then - file="$(cat "$__object/parameter/file")" + file=$(cat "$__object/parameter/file") else file="/$__object_id" fi @@ -55,41 +56,45 @@ function _find(_text, _pattern) { BEGIN { getline anchor < (ENVIRON["__object"] "/parameter/" position) getline pattern < (ENVIRON["__object"] "/parameter/" needle) - state = "absent" + + found_line = 0 + correct_pos = (position != "after" && position != "before") } { if (position == "after") { if (match($0, anchor)) { getline if (_find($0, pattern)) { - state = "present" + found_line++ + correct_pos = 1 + exit 0 } - else { - state = "wrongposition" - } - exit 0 + } else if (_find($0, pattern)) { + found_line++ } - } - else if (position == "before") { + } else if (position == "before") { if (_find($0, pattern)) { + found_line++ getline if (match($0, anchor)) { - state = "present" + correct_pos = 1 + exit 0 } - else { - state = "wrongposition" - } - exit 0 } - } - else { + } else { if (_find($0, pattern)) { - state = "present" + found_line++ exit 0 } } } END { - print state + if (found_line && correct_pos) { + print "present" + } else if (found_line) { + print "wrongposition" + } else { + print "absent" + } } ' "$file" From cd2d5b3f79d1e3ea8545b67096d67d6ceda763e6 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Jan 2020 19:24:21 +0200 Subject: [PATCH 062/138] __apt_unattended_upgrades: all objects depend on package --- cdist/conf/type/__apt_unattended_upgrades/manifest | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index 88a5ccd8..811e79a6 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -20,16 +20,17 @@ __package unattended-upgrades +export require='__package/unattended-upgrades' + # in normal circumstances 20auto-upgrades is managed # by debconf and it can only contain these lines # https://wiki.debian.org/UnattendedUpgrades -require='__package/unattended-upgrades' \ - __file /etc/apt/apt.conf.d/20auto-upgrades \ - --owner root \ - --group root \ - --mode 644 \ - --source - << EOF +__file /etc/apt/apt.conf.d/20auto-upgrades \ + --owner root \ + --group root \ + --mode 644 \ + --source - << EOF APT::Periodic::Update-Package-Lists "1"; APT::Periodic::Unattended-Upgrade "1"; EOF From 68e5502fce3d3ae043ae0792a14abd1970834d36 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Jan 2020 19:27:02 +0200 Subject: [PATCH 063/138] __apt_unattended_upgrades: move debian wiki link to manual --- cdist/conf/type/__apt_unattended_upgrades/man.rst | 2 ++ cdist/conf/type/__apt_unattended_upgrades/manifest | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/man.rst b/cdist/conf/type/__apt_unattended_upgrades/man.rst index d64b2e9e..f75c6513 100644 --- a/cdist/conf/type/__apt_unattended_upgrades/man.rst +++ b/cdist/conf/type/__apt_unattended_upgrades/man.rst @@ -11,6 +11,8 @@ DESCRIPTION Install and configure unattended-upgrades package. +For more information see https://wiki.debian.org/UnattendedUpgrades. + OPTIONAL PARAMETERS ------------------- diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index 811e79a6..03cc2c50 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -24,7 +24,6 @@ export require='__package/unattended-upgrades' # in normal circumstances 20auto-upgrades is managed # by debconf and it can only contain these lines -# https://wiki.debian.org/UnattendedUpgrades __file /etc/apt/apt.conf.d/20auto-upgrades \ --owner root \ From 2cf44c66d46435e859fe0528d4b414dafd8b468a Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 15 Jan 2020 19:00:30 +0100 Subject: [PATCH 064/138] [__directory] Add --state exists and --state pre-exists --- cdist/conf/type/__directory/gencode-remote | 17 ++++++++++++++--- cdist/conf/type/__directory/man.rst | 15 +++++++++++++-- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 374db47a..e1ab69d7 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -3,6 +3,7 @@ # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Steven Armstrong (steven-cdist armstrong.cc) # 2014 Daniel Heule (hda at sfs.biz) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -21,8 +22,8 @@ # destination="/$__object_id" -state_should="$(cat "$__object/parameter/state")" -type="$(cat "$__object/explorer/type")" +state_should=$(cat "$__object/parameter/state") +type=$(cat "$__object/explorer/type") stat_file="$__object/explorer/stat" # variable to keep track if we have to set directory attributes @@ -72,7 +73,7 @@ set_mode() { } case "$state_should" in - present) + present|exists) if [ "$type" != "directory" ]; then set_attributes=1 if [ "$type" != "none" ]; then @@ -83,6 +84,10 @@ case "$state_should" in fi echo "mkdir $mkdiropt '$destination'" echo "create" >> "$__messages_out" + elif [ "$state_should" = 'exists' ]; then + # The type is directory and --state exists. We are done and do not + # check or set the attributes. + exit 0 fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by @@ -103,6 +108,12 @@ case "$state_should" in fi done ;; + pre-exists) + if [ "$type" != "directory" ]; then + echo "Directory \"$destination\" does not exist" >&2 + exit 1 + fi + ;; absent) if [ "$type" = "directory" ]; then echo "rm -rf '$destination'" diff --git a/cdist/conf/type/__directory/man.rst b/cdist/conf/type/__directory/man.rst index 74b00afe..7755334c 100644 --- a/cdist/conf/type/__directory/man.rst +++ b/cdist/conf/type/__directory/man.rst @@ -19,7 +19,18 @@ None. OPTIONAL PARAMETERS ------------------- state - 'present' or 'absent', defaults to 'present' + 'present', 'absent', 'exists' or 'pre-exists', defaults to 'present' where: + + present + the directory exists and the given attributes are set. + absent + the directory does not exist. + exists + the directory exists, but its attributes are not altered if it already + existed. + pre-exists + check that the directory exists and is indeed a directory, but do not + create or modify it. group Group to chgrp to. @@ -36,7 +47,7 @@ BOOLEAN PARAMETERS parents Whether to create parents as well (mkdir -p behaviour). Warning: all intermediate directory permissions default - to whatever mkdir -p does. + to whatever mkdir -p does. Usually this means root:root, 0700. From cd24a806e7be7b3300543943e8c7a5582a55d15e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Jan 2020 20:48:32 +0200 Subject: [PATCH 065/138] __apt_unattended_upgrades: rewrite manifest, add more parameters --- .../type/__apt_unattended_upgrades/manifest | 88 +++++++++++++------ .../parameter/boolean | 4 + 2 files changed, 63 insertions(+), 29 deletions(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index 03cc2c50..cb99b611 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -34,39 +34,61 @@ APT::Periodic::Update-Package-Lists "1"; APT::Periodic::Unattended-Upgrade "1"; EOF -# lets not write into upstream 50unattended-upgrades file, -# but use our own config files to avoid clashes - -ml_conf='/etc/apt/apt.conf.d/51unattended-upgrades-mail' -bl_conf='/etc/apt/apt.conf.d/51unattended-upgrades-blacklist' +conf='# this file is managed by cdist' if [ -f "$__object/parameter/mail" ] then - mail="$( cat "$__object/parameter/mail" )" -else - mail='' + conf="$( + printf \ + '%s\nUnattended-Upgrade::Mail "%s";\n' \ + "$conf" \ + "$( cat "$__object/parameter/mail" )" + )" fi -if [ -n "$mail" ] +if [ -f "$__object/parameter/mail-on-error" ] then - if [ -f "$__object/parameter/mail-on-error" ] - then - mail_on_error='true' - else - mail_on_error='false' - fi + conf="$( + printf \ + '%s\nUnattended-Upgrade::MailOnlyOnError "true";\n' \ + "$conf" + )" +fi - __file "$ml_conf" \ - --owner root \ - --group root \ - --mode 644 \ - --source - << EOF -Unattended-Upgrade::Mail "$mail"; -Unattended-Upgrade::MailOnlyOnError "$mail_on_error"; -EOF +if [ -f "$__object/parameter/no-auto-fix" ] +then + conf="$( + printf \ + '%s\nUnattended-Upgrade::AutoFixInterruptedDpkg "false";\n' \ + "$conf" + )" +fi -else - __file "$ml_conf" --state absent +if [ -f "$__object/parameter/no-minimal-steps" ] +then + conf="$( + printf \ + '%s\nUnattended-Upgrade::MinimalSteps "false";\n' \ + "$conf" + )" +fi + +if [ -f "$__object/parameter/on-shutdown" ] +then + conf="$( + printf \ + '%s\nUnattended-Upgrade::InstallOnShutdown "true";\n' \ + "$conf" + )" +fi + +if [ -f "$__object/parameter/reboot" ] +then + conf="$( + printf \ + '%s\nUnattended-Upgrade::Automatic-Reboot "true";\n' \ + "$conf" + )" fi if [ -f "$__object/parameter/blacklist" ] @@ -79,14 +101,22 @@ then done \ < "$__object/parameter/blacklist" - bl="$( printf '%s\n}' "$bl" )" + conf="$( printf '%s\n%s\n}\n' "$conf" "$bl" )" +fi - echo "$bl" \ - | __file "$bl_conf" \ +# lets not write into upstream 50unattended-upgrades file, +# but use our own config files to avoid clashes + +conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist' + +if [ "$( echo "$conf" | wc -l )" -gt 1 ] +then + echo "$conf" \ + | __file "$conf_file" \ --owner root \ --group root \ --mode 644 \ --source - else - __file "$bl_conf" --state absent + __file "$conf_file" --state absent fi diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean index edcaa12a..6ad9790f 100644 --- a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean +++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean @@ -1 +1,5 @@ mail-on-error +no-auto-fix +no-minimal-steps +on-shutdown +reboot From b3f36dbe5bc7d1e21892f765ee08dfc5d8c6a6e3 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Jan 2020 20:51:47 +0200 Subject: [PATCH 066/138] __apt_unattended_upgrades: fix typo --- cdist/conf/type/__apt_unattended_upgrades/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index cb99b611..04225a9b 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -105,7 +105,7 @@ then fi # lets not write into upstream 50unattended-upgrades file, -# but use our own config files to avoid clashes +# but use our own config file to avoid clashes conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist' From 5a9a1ba57fc45df63cff123a42905360c3a7bddb Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 15 Jan 2020 22:00:56 +0100 Subject: [PATCH 067/138] [__line] Produce error when file does not exist --- cdist/conf/type/__line/explorer/state | 5 +---- cdist/conf/type/__line/gencode-remote | 17 +++++++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index 6ff0a798..e8fc3630 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -25,10 +25,7 @@ else file="/$__object_id" fi -if [ ! -f "$file" ]; then - echo "file_missing" - exit 0 -fi +[ -f "$file" ] || exit 0 if [ -f "$__object/parameter/before" ]; then position="before" diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index c8c90c38..88cae68b 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -24,9 +24,20 @@ if [ -f "$__object/parameter/before" ] && [ -f "$__object/parameter/after" ]; th exit 1 fi +if [ -f "$__object/parameter/file" ]; then + file="$(cat "$__object/parameter/file")" +else + file="/$__object_id" +fi + state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" +if [ -z "$state_is" ]; then + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 +fi + if [ "$state_should" = "$state_is" ]; then # nothing to do exit 0 @@ -47,12 +58,6 @@ else needle="line" fi -if [ -f "$__object/parameter/file" ]; then - file="$(cat "$__object/parameter/file")" -else - file="/$__object_id" -fi - add=0 remove=0 case "$state_should" in From f5f70671cb3e4ca8b2d5e2a0e66a1847f05ef4bf Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 16 Jan 2020 00:59:03 +0200 Subject: [PATCH 068/138] __apt_unattended_upgrades: make parameters more speaking --- cdist/conf/type/__apt_unattended_upgrades/manifest | 8 ++++---- .../conf/type/__apt_unattended_upgrades/parameter/boolean | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index 04225a9b..094dfa1c 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -46,7 +46,7 @@ then )" fi -if [ -f "$__object/parameter/mail-on-error" ] +if [ -f "$__object/parameter/mail-only-on-error" ] then conf="$( printf \ @@ -55,7 +55,7 @@ then )" fi -if [ -f "$__object/parameter/no-auto-fix" ] +if [ -f "$__object/parameter/no-auto-fix-interrupted-dpkg" ] then conf="$( printf \ @@ -73,7 +73,7 @@ then )" fi -if [ -f "$__object/parameter/on-shutdown" ] +if [ -f "$__object/parameter/install-on-shutdown" ] then conf="$( printf \ @@ -82,7 +82,7 @@ then )" fi -if [ -f "$__object/parameter/reboot" ] +if [ -f "$__object/parameter/automatic-reboot" ] then conf="$( printf \ diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean index 6ad9790f..831dc95c 100644 --- a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean +++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean @@ -1,5 +1,5 @@ -mail-on-error -no-auto-fix +mail-only-on-error +no-auto-fix-interrupted-dpkg no-minimal-steps -on-shutdown -reboot +install-on-shutdown +automatic-reboot From 6fa0d687f7bdf3ba1e6a032118553d90ceba0b22 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 16 Jan 2020 01:13:27 +0200 Subject: [PATCH 069/138] __apt_unattended_upgrades: alternative approach --- .../type/__apt_unattended_upgrades/manifest | 100 ++++++------------ .../parameter/boolean | 5 - .../parameter/optional | 1 - .../parameter/optional_multiple | 1 + 4 files changed, 31 insertions(+), 76 deletions(-) delete mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/boolean delete mode 100644 cdist/conf/type/__apt_unattended_upgrades/parameter/optional diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index 094dfa1c..2f8f6e76 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -34,81 +34,41 @@ APT::Periodic::Update-Package-Lists "1"; APT::Periodic::Unattended-Upgrade "1"; EOF -conf='# this file is managed by cdist' - -if [ -f "$__object/parameter/mail" ] -then - conf="$( - printf \ - '%s\nUnattended-Upgrade::Mail "%s";\n' \ - "$conf" \ - "$( cat "$__object/parameter/mail" )" - )" -fi - -if [ -f "$__object/parameter/mail-only-on-error" ] -then - conf="$( - printf \ - '%s\nUnattended-Upgrade::MailOnlyOnError "true";\n' \ - "$conf" - )" -fi - -if [ -f "$__object/parameter/no-auto-fix-interrupted-dpkg" ] -then - conf="$( - printf \ - '%s\nUnattended-Upgrade::AutoFixInterruptedDpkg "false";\n' \ - "$conf" - )" -fi - -if [ -f "$__object/parameter/no-minimal-steps" ] -then - conf="$( - printf \ - '%s\nUnattended-Upgrade::MinimalSteps "false";\n' \ - "$conf" - )" -fi - -if [ -f "$__object/parameter/install-on-shutdown" ] -then - conf="$( - printf \ - '%s\nUnattended-Upgrade::InstallOnShutdown "true";\n' \ - "$conf" - )" -fi - -if [ -f "$__object/parameter/automatic-reboot" ] -then - conf="$( - printf \ - '%s\nUnattended-Upgrade::Automatic-Reboot "true";\n' \ - "$conf" - )" -fi - -if [ -f "$__object/parameter/blacklist" ] -then - bl='Unattended-Upgrade::Package-Blacklist {'; - - while read -r l - do - bl="$( printf '%s\n"%s";\n' "$bl" "$l" )" - done \ - < "$__object/parameter/blacklist" - - conf="$( printf '%s\n%s\n}\n' "$conf" "$bl" )" -fi - # lets not write into upstream 50unattended-upgrades file, # but use our own config file to avoid clashes conf_file='/etc/apt/apt.conf.d/51unattended-upgrades-cdist' +conf='# this file is managed by cdist' + +if [ -f "$__object/parameter/option" ] +then + o='' + + while read -r l + do + k="$( echo "$l" | awk -F= '{print $1}' )" + v="$( echo "$l" | awk -F= '{print $2}' )" + o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "$k" "$v" )" + done \ + < "$__object/parameter/option" + + conf="$( printf '%s\n%s\n' "$conf" "$o" )" +fi + +if [ -f "$__object/parameter/blacklist" ] +then + b='Unattended-Upgrade::Package-Blacklist {' + + while read -r l + do + b="$( printf '%s\n"%s";\n' "$b" "$l" )" + done \ + < "$__object/parameter/blacklist" + + conf="$( printf '%s\n%s\n}\n' "$conf" "$b" )" +fi + if [ "$( echo "$conf" | wc -l )" -gt 1 ] then echo "$conf" \ diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean b/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean deleted file mode 100644 index 831dc95c..00000000 --- a/cdist/conf/type/__apt_unattended_upgrades/parameter/boolean +++ /dev/null @@ -1,5 +0,0 @@ -mail-only-on-error -no-auto-fix-interrupted-dpkg -no-minimal-steps -install-on-shutdown -automatic-reboot diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional deleted file mode 100644 index fa7963cc..00000000 --- a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional +++ /dev/null @@ -1 +0,0 @@ -mail diff --git a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple index 27b9ffc9..ea4fba2b 100644 --- a/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple +++ b/cdist/conf/type/__apt_unattended_upgrades/parameter/optional_multiple @@ -1 +1,2 @@ +option blacklist From ef2f4b9a004369cc390f1271721bc22d6a44f402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 17 Jan 2020 11:21:28 +0100 Subject: [PATCH 070/138] __postgres_*: fix forgotten edge cases in delimited identifier escape --- cdist/conf/type/__postgres_database/gencode-remote | 8 ++++++-- cdist/conf/type/__postgres_role/gencode-remote | 10 ++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 9f12d215..47e6b97c 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -43,10 +43,14 @@ if [ "$state_should" != "$state_is" ]; then if [ -f "$__object/parameter/owner" ]; then owner="-O \"$(cat "$__object/parameter/owner")\"" fi - echo "su - '$postgres_user' -c \"createdb $owner \"$name\"\"" + cat << EOF +su - '$postgres_user' -c "createdb $owner \"$name\"" +EOF ;; absent) - echo "su - '$postgres_user' -c \"dropdb \"$name\"\"" + cat << EOF +su - '$postgres_user' -c "dropdb \"$name\"" +EOF ;; esac fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 0b8852f4..977832c9 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -53,11 +53,13 @@ case "$state_should" in done [ -n "$password" ] && password="PASSWORD '$password'" - - cmd="CREATE ROLE \"$name\" WITH $password $booleans" - echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\"" + cat << EOF +su - '$postgres_user' -c "psql postgres -wc 'CREATE ROLE \"$name\" WITH $password $booleans;'" +EOF ;; absent) - echo "su - '$postgres_user' -c \"dropuser \\\"$name\\\"\"" + cat << EOF +su - '$postgres_user' -c "dropuser \"$name\"" +EOF ;; esac From 6097cd9e90341146948886a9480cd0b70ee9e60f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 17 Jan 2020 14:41:00 +0200 Subject: [PATCH 071/138] __apt_unattended_upgrades: use word expansions --- cdist/conf/type/__apt_unattended_upgrades/manifest | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/manifest b/cdist/conf/type/__apt_unattended_upgrades/manifest index 2f8f6e76..3c00e2f4 100755 --- a/cdist/conf/type/__apt_unattended_upgrades/manifest +++ b/cdist/conf/type/__apt_unattended_upgrades/manifest @@ -47,9 +47,7 @@ then while read -r l do - k="$( echo "$l" | awk -F= '{print $1}' )" - v="$( echo "$l" | awk -F= '{print $2}' )" - o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "$k" "$v" )" + o="$( printf '%s\nUnattended-Upgrade::%s "%s";\n' "$o" "${l%%=*}" "${l#*=}" )" done \ < "$__object/parameter/option" From 65c5b592a590f3f4f8b31a1b7c32b2a98354d054 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 17 Jan 2020 14:48:54 +0200 Subject: [PATCH 072/138] __apt_unattended_upgrades: update manual --- .../type/__apt_unattended_upgrades/man.rst | 40 ++++++++++++------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__apt_unattended_upgrades/man.rst b/cdist/conf/type/__apt_unattended_upgrades/man.rst index f75c6513..2231b5f9 100644 --- a/cdist/conf/type/__apt_unattended_upgrades/man.rst +++ b/cdist/conf/type/__apt_unattended_upgrades/man.rst @@ -14,35 +14,47 @@ Install and configure unattended-upgrades package. For more information see https://wiki.debian.org/UnattendedUpgrades. -OPTIONAL PARAMETERS -------------------- -mail - Send email to this address for problems or packages upgrades. - - OPTIONAL MULTIPLE PARAMETERS ---------------------------- +option + Set options for unattended-upgrades. See examples. + + Supported options with default values (as of 2020-01-17) are: + + - AutoFixInterruptedDpkg, default is "true" + - MinimalSteps, default is "true" + - InstallOnShutdown, default is "false" + - Mail, default is "" (empty) + - MailOnlyOnError, default is "false" + - Remove-Unused-Kernel-Packages, default is "true" + - Remove-New-Unused-Dependencies, default is "true" + - Remove-Unused-Dependencies, default is "false" + - Automatic-Reboot, default is "false" + - Automatic-Reboot-WithUsers, default is "true" + - Automatic-Reboot-Time, default is "02:00" + - SyslogEnable, default is "false" + - SyslogFacility, default is "daemon" + - OnlyOnACPower, default is "true" + - Skip-Updates-On-Metered-Connections, default is "true" + - Verbose, default is "false" + - Debug, default is "false" + blacklist Python regular expressions, matching packages to exclude from upgrading. -BOOLEAN PARAMETERS ------------------- -mail-on-error - Get emails only on errors. - - EXAMPLES -------- .. code-block:: sh __apt_unattended_upgrades \ - --mail root \ - --mail-on-error \ + --option Mail=root \ + --option MailOnlyOnError=true \ --blacklist multipath-tools \ --blacklist open-iscsi + AUTHORS ------- Ander Punnar From e292c9f1245a59cda5c721d4ce4e201ce9799349 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 17 Jan 2020 15:48:59 +0100 Subject: [PATCH 073/138] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 526fc320..bc251d64 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,8 @@ next: * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) * Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak) * Core info command: Support tilde expansion of conf directories (Darko Poljak) + * Types __postgres_*: Fix edge cases in quoted identifiers (Timothée Floure) + * New type: __apt_unattended_upgrades (Ander Punnar) 6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) From 45890cc7e43577252e8560ca70611a88e9936ffa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 17 Jan 2020 18:14:49 +0100 Subject: [PATCH 074/138] ++changelog --- docs/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog b/docs/changelog index bc251d64..ccaf848f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,12 @@ next: * Core info command: Support tilde expansion of conf directories (Darko Poljak) * Types __postgres_*: Fix edge cases in quoted identifiers (Timothée Floure) * New type: __apt_unattended_upgrades (Ander Punnar) + * Type __line: Bugfixes: (Dennis Camera) + - ensure the line is only added once + - always add line to end if anchor is not found + - match line at the beginning when not regex + - fix incorrect 'wrongposition' in state explorer + - produce error when file does not exist 6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) From e1ac97b6a5b10d29b7b9bd70c5adea7e9c33b2e9 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 19 Jan 2020 14:02:00 +0200 Subject: [PATCH 075/138] __acl: add --source --- cdist/conf/type/__acl/gencode-remote | 12 +++++++++++- cdist/conf/type/__acl/man.rst | 12 ++++++++++++ cdist/conf/type/__acl/parameter/optional | 1 + 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index f4f0d1e2..70a46af5 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -28,7 +28,17 @@ acl_path="/$__object_id" acl_is="$( cat "$__object/explorer/acl_is" )" -if [ -f "$__object/parameter/entry" ] +if [ -f "$__object/parameter/source" ] +then + acl_source="$( cat "$__object/parameter/source" )" + + if [ "$acl_source" = '-' ] + then + acl_should="$( cat "$__object/stdin" )" + else + acl_should="$( grep -Ev '^#|^$' "$acl_source" )" + fi +elif [ -f "$__object/parameter/entry" ] then acl_should="$( cat "$__object/parameter/entry" )" elif [ -f "$__object/parameter/acl" ] diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index c3493e49..e7ef9579 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -19,6 +19,14 @@ entry Set ACL entry following ``getfacl`` output syntax. +OPTIONAL PARAMETERS +------------------- +source + Read ACL entries from stdin or file. + Ordering of entries is not important. + When reading from file, comments and empty lines are ignored. + + BOOLEAN PARAMETERS ------------------ default @@ -71,6 +79,10 @@ EXAMPLES --entry group:secret-project:rwx \ --entry user:alice:r-x + # read acl from stdin + echo 'user:alice:rwx' \ + | __acl /path/to/directory --source - + AUTHORS ------- diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional index 4b32086b..12edcccb 100644 --- a/cdist/conf/type/__acl/parameter/optional +++ b/cdist/conf/type/__acl/parameter/optional @@ -1,2 +1,3 @@ mask other +source From 5089f9055ecb0b826942c8f947eb26f9ef397aaa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 19 Jan 2020 19:41:26 +0100 Subject: [PATCH 076/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ccaf848f..3f240e5a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,7 @@ next: - match line at the beginning when not regex - fix incorrect 'wrongposition' in state explorer - produce error when file does not exist + * Type __acl: Add --source parameter (Ander Punnar) 6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) From 1fbd2fc2bdf76697dd50808cd3927d15f6b8bc5c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 19 Jan 2020 22:51:40 +0200 Subject: [PATCH 077/138] __acl: add --file and --directory for convenience --- cdist/conf/type/__acl/gencode-remote | 8 +++++++- cdist/conf/type/__acl/man.rst | 13 +++++++++++++ cdist/conf/type/__acl/manifest | 11 +++++++++++ cdist/conf/type/__acl/parameter/optional | 2 ++ 4 files changed, 33 insertions(+), 1 deletion(-) create mode 100755 cdist/conf/type/__acl/manifest diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote index 70a46af5..e5404a9d 100755 --- a/cdist/conf/type/__acl/gencode-remote +++ b/cdist/conf/type/__acl/gencode-remote @@ -20,7 +20,13 @@ file_is="$( cat "$__object/explorer/file_is" )" -[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0 +if [ "$file_is" = 'missing' ] \ + && [ -z "$__cdist_dry_run" ] \ + && \( [ ! -f "$__object/parameter/file" ] \ + || [ ! -f "$__object/parameter/directory" ] \) +then + exit 0 +fi os="$( cat "$__global/explorer/os" )" diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst index e7ef9579..28412871 100644 --- a/cdist/conf/type/__acl/man.rst +++ b/cdist/conf/type/__acl/man.rst @@ -26,6 +26,12 @@ source Ordering of entries is not important. When reading from file, comments and empty lines are ignored. +file + Create/change file with ``__file`` using ``user:group:mode`` pattern. + +directory + Create/change directory with ``__directory`` using ``user:group:mode`` pattern. + BOOLEAN PARAMETERS ------------------ @@ -83,6 +89,13 @@ EXAMPLES echo 'user:alice:rwx' \ | __acl /path/to/directory --source - + # create/change directory too + __acl /path/to/directory \ + --default \ + --remove \ + --directory root:root:770 \ + --entry user:nobody:rwx + AUTHORS ------- diff --git a/cdist/conf/type/__acl/manifest b/cdist/conf/type/__acl/manifest new file mode 100755 index 00000000..5fd23110 --- /dev/null +++ b/cdist/conf/type/__acl/manifest @@ -0,0 +1,11 @@ +#!/bin/sh -e + +for p in file directory +do + [ ! -f "$__object/parameter/$p" ] && continue + + "__$p" "/$__object_id" \ + --owner "$( awk -F: '{print $1}' "$__object/parameter/$p" )" \ + --group "$( awk -F: '{print $2}' "$__object/parameter/$p" )" \ + --mode "$( awk -F: '{print $3}' "$__object/parameter/$p" )" +done diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional index 12edcccb..cdcbc0b8 100644 --- a/cdist/conf/type/__acl/parameter/optional +++ b/cdist/conf/type/__acl/parameter/optional @@ -1,3 +1,5 @@ mask other source +file +directory From 4e8ccd9b788c11403f586bf0c413d57546013c1b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 20 Jan 2020 09:08:55 +0100 Subject: [PATCH 078/138] ++changelog --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3f240e5a..083cf5fb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,7 +13,7 @@ next: - match line at the beginning when not regex - fix incorrect 'wrongposition' in state explorer - produce error when file does not exist - * Type __acl: Add --source parameter (Ander Punnar) + * Type __acl: Add --source, --file and --directory parameters (Ander Punnar) 6.4.0: 2020-01-04 * Type __consul_agent: Don't deploy init script on Alpine anymore, it ships with one itself (Nico Schottelius) From 8f12a4c505812d640cd926567ee8c6300a6341fb Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 23 Jan 2020 07:10:48 +0100 Subject: [PATCH 079/138] Release 6.5.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 083cf5fb..467c0f22 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) * Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak) * Core info command: Support tilde expansion of conf directories (Darko Poljak) From 87f30b6053315f11013bbf92ce1281bca2d54f43 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 23 Jan 2020 14:40:07 +0100 Subject: [PATCH 080/138] Update sphinx docs copyright year --- docs/src/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 78f9842c..47765413 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2019' +copyright = 'ungleich GmbH 2020' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for From de1a421b68edeb2bf3660f2ed98e109db93e8d88 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 2 Oct 2019 15:39:30 +0200 Subject: [PATCH 081/138] [explorer/init] Support for Darwin and more BusyBox combinations --- cdist/conf/explorer/init | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index a8a7857e..829d6ab8 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -23,14 +23,22 @@ # for example at linux this value is "init" or "systemd" in most cases # -uname_s="$(uname -s)" - -case "$uname_s" in +case $(uname -s) in Linux) - (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true + if command -v pgrep >/dev/null + then + # BusyBox's version of ps does not support some options. + # On Linux systems, we prefer pgrep to get the name of PID1. + (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true + else + ps -o comm= -p 1 2>/dev/null || cat /proc/1/comm + fi ;; FreeBSD|OpenBSD) - ps -o comm= -p 1 || true + ps -o comm= -p 1 2>/dev/null || true + ;; + Darwin) + basename "$(ps -o comm= -p 1 2>/dev/null)" ;; *) # return a empty string as unknown value From 4fe2dcba891ae52cccbe02553976be2bb7291fd1 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 30 Jan 2020 18:35:50 +0100 Subject: [PATCH 082/138] [explorer/init] Linux is a mess... --- cdist/conf/explorer/init | 44 +++++++++++++++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index 829d6ab8..ceae2e9f 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -25,14 +25,48 @@ case $(uname -s) in Linux) - if command -v pgrep >/dev/null + if test -d /proc/1/ then - # BusyBox's version of ps does not support some options. - # On Linux systems, we prefer pgrep to get the name of PID1. - (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true + comm_name=$(cat /proc/1/comm) else - ps -o comm= -p 1 2>/dev/null || cat /proc/1/comm + # BusyBox's versions of ps and pgrep do not support some options + # depending on which compile-time options have been used. + # Both pgrep and ps are tried to get the command name + comm_name=$( + pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }' + || ps -o comm= -p 1 2>/dev/null) fi + + case $comm_name + in + systemd) + echo systemd + ;; + init) + # It could be anything... + + if test -h /proc/1/exe + then + init_exe=/proc/1/exe + else + init_exe=$(command -v "$comm_name") + fi + + test -x "$comm_exe" || exit 1 + + case $("$comm_exe" --version | head -n 1) + in + *SysV*) + echo init + ;; + *upstart*) + echo upstart + ;; + *) + echo "" + ;; + esac + esac ;; FreeBSD|OpenBSD) ps -o comm= -p 1 2>/dev/null || true From e5d86ffc9360767502469d00a0db1aca72d4f483 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 1 Feb 2020 17:08:25 +0100 Subject: [PATCH 083/138] Fix to remove cronjobs when the cronjob expression did not match. --- cdist/conf/type/__cron/gencode-remote | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote index 59398058..8f3c92dc 100755 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -42,8 +42,12 @@ fi mkdir "$__object/files" echo "$entry" > "$__object/files/entry" -if diff -q "$__object/files/entry" "$__object/explorer/entry" >/dev/null; then - state_is=present +if [ -s "$__object/explorer/entry" ]; then + if diff -q "$__object/files/entry" "$__object/explorer/entry" >/dev/null; then + state_is=present + else + state_is=modified + fi else state_is=absent fi From f3237e5d76c995d138bdf287c00a03d48f4ac742 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 3 Feb 2020 17:44:47 +0100 Subject: [PATCH 084/138] [consul agent] add support for Debian 10 --- cdist/conf/type/__consul_agent/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index ee682d72..e00f29ec 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -209,7 +209,7 @@ case "$os" in [567]) init_sysvinit debian ;; - [89]) + [89]|10) init_systemd ;; *) From 0f420993e1befd2692aca6cae43efb751a4a78de Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 3 Feb 2020 17:45:34 +0100 Subject: [PATCH 085/138] ++ changelog update --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 467c0f22..9f5e73bd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __consul_agent: Add Debian 10 support (Nico Schottelius) + 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) * Core: preos: Fix missing configuration file usage, support -g, --config-file option (Darko Poljak) From e82dd35869e00b2fb00b6907b5fc3a960981bc6e Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 30 Sep 2019 18:50:20 +0200 Subject: [PATCH 086/138] [__file] stat explorer patch for systems without stat(1) Some embedded systems (like OpenWrt) do not ship a stat(1) binary. This workaround parses the output of ls(1) and /etc/passwd, /etc/group to gather the information needed. --- cdist/conf/type/__file/explorer/stat | 64 ++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 13c1c208..91c8cc84 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -2,6 +2,7 @@ # # 2013 Steven Armstrong (steven-cdist armstrong.cc) # 2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -21,29 +22,54 @@ destination="/$__object_id" +fallback() { + # Fallback: Patch the output together, manually. + + ls_line=$(ls -ldn "$destination") + + uid=$(echo "$ls_line" | awk '{ print $3 }') + gid=$(echo "$ls_line" | awk '{ print $4 }') + + owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd) + group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) + + mode_text=$(echo "$ls_line" | awk '{ print $1 }') + mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }') + + size=$(echo "$ls_line" | awk '{ print $5 }') + links=$(echo "$ls_line" | awk '{ print $2 }') + + printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\nsize: %d\nlinks: %d\n' \ + "$("$__type_explorer/type")" \ + "$uid" "$owner" \ + "$gid" "$group" \ + "$mode" "$mode_text" \ + "$size" \ + "$links" +} + + # nothing to work with, nothing we could do [ -e "$destination" ] || exit 0 -os=$("$__explorer/os") -case "$os" in - "freebsd"|"netbsd"|"openbsd"|"macosx") + +if ! command -v stat >/dev/null +then + fallback + exit +fi + + +case $("$__explorer/os") +in + freebsd|netbsd|openbsd|macosx) stat -f "type: %HT owner: %Du %Su group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl -" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' - ;; - alpine) - # busybox stat - stat -c "type: %F -owner: %u %U -group: %g %G -mode: %a %A -size: %s -links: %h -" "$destination" +" "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; solaris) ls1="$( ls -ld "$destination" )" @@ -77,12 +103,14 @@ links: %h echo "links: $( echo "$ls1" | awk '{print $2}' )" ;; *) - stat --printf="type: %F + # NOTE: Do not use --printf here as it is not supported by BusyBox stat. + # NOTE: BusyBox's stat might not support the "-c" option, in which case + # we fall through to the shell fallback. + stat -c "type: %F owner: %u %U group: %g %G mode: %a %A size: %s -links: %h -" "$destination" - ;; +links: %h" "$destination" 2>/dev/null || fallback + ;; esac From 3b5433d63af27f06f3c44b121b21d7e0520af7bf Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 3 Feb 2020 18:12:27 +0100 Subject: [PATCH 087/138] [__directory] stat explorer patch for systems without stat(1) Some embedded systems (like OpenWrt) do not ship a stat(1) binary. This workaround parses the output of ls(1) and /etc/passwd, /etc/group to gather the information needed. --- cdist/conf/type/__directory/explorer/stat | 48 +++++++++++++++++------ 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat index 03d466ba..105d894f 100755 --- a/cdist/conf/type/__directory/explorer/stat +++ b/cdist/conf/type/__directory/explorer/stat @@ -1,6 +1,7 @@ #!/bin/sh # # 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -20,24 +21,43 @@ destination="/$__object_id" +fallback() { + # Patch the output together, manually + + ls_line=$(ls -ldn "$destination") + + uid=$(echo "$ls_line" | awk '{ print $3 }') + gid=$(echo "$ls_line" | awk '{ print $4 }') + + owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd) + group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) + + mode_text=$(echo "$ls_line" | awk '{ print $1 }') + mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }') + + printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\n' \ + "$("$__type_explorer/type")" \ + "$uid" "$owner" \ + "$gid" "$group" \ + "$mode" "$mode_text" +} + # nothing to work with, nothing we could do [ -e "$destination" ] || exit 0 -os=$("$__explorer/os") -case "$os" in +if ! command -v stat >/dev/null +then + fallback + exit +fi + +case $("$__explorer/os") in "freebsd"|"netbsd"|"openbsd"|"macosx") stat -f "type: %HT owner: %Du %Su group: %Dg %Sg mode: %Lp %Sp -" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' - ;; - alpine) - stat -c "type: %F -owner: %u %U -group: %g %G -mode: %a %A -" "$destination" +" "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; solaris) ls1="$( ls -ld "$destination" )" @@ -69,10 +89,12 @@ mode: %a %A echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" ;; *) - stat --printf="type: %F + # NOTE: Do not use --printf here as it is not supported by BusyBox stat. + # NOTE: BusyBox's stat might not support the "-c" option, in which case + # we fall through to the shell fallback. + stat -c "type: %F owner: %u %U group: %g %G -mode: %a %A -" "$destination" +mode: %a %A" "$destination" 2>/dev/null || fallback ;; esac From 984e0dc8c4975904057a7ab0a8c9d357ed0b4fdf Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 3 Feb 2020 19:29:04 +0100 Subject: [PATCH 088/138] [explorer/os_release] Add fallbacks to /usr/lib/os-release and /var/run/os-release --- cdist/conf/explorer/os_release | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index cfc01004..6489446b 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -1,6 +1,7 @@ #!/bin/sh # # 2018 Adam Dej (dejko.a at gmail.com) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -21,6 +22,17 @@ # See os-release(5) and http://0pointer.de/blog/projects/os-release -set +e +if test -f /etc/os-release +then + # Linux and FreeBSD (usually a symlink) + cat /etc/os-release +elif test -f /usr/lib/os-release +then + # systemd + cat /usr/lib/os-release +elif test -f /var/run/os-release +then + # FreeBSD (created by os-release service) + cat /var/run/os-release +fi -cat /etc/os-release || cat /usr/lib/os-release || true From 7a5d6d5a7d602863cd7d74bfb4e13a8bb59c5944 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 4 Feb 2020 07:45:04 +0100 Subject: [PATCH 089/138] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9f5e73bd..bf0a8777 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * Type __consul_agent: Add Debian 10 support (Nico Schottelius) + * Explorer os_release: Add fallbacks (Dennis Camera) + * Types __file, __directory: Add fallback for systems without stat (Dennis Camera) 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) From 08d5814e2cc8ea6a204afca91bec7b831116112b Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 9 Feb 2020 20:37:06 +0200 Subject: [PATCH 090/138] __package_apt: update index cache, on installation, when it is older than one day --- cdist/conf/type/__package_apt/gencode-remote | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index 699eb0c9..b3184a9c 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -74,6 +74,11 @@ fi case "$state_should" in present) + cat << EOF +if [ -f /var/cache/apt/pkgcache.bin ] && [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] +then echo apt-get update > /dev/null 2>&1 || true +fi +EOF if [ -n "$version" ]; then name="${name}=${version}" fi From b3bad9468de78cf90e8ce92a59c8bb3d285883fe Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 9 Feb 2020 20:43:09 +0200 Subject: [PATCH 091/138] __package_apt: pkgcache.bin may not exist --- cdist/conf/type/__package_apt/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index b3184a9c..cb79e886 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -75,7 +75,7 @@ fi case "$state_should" in present) cat << EOF -if [ -f /var/cache/apt/pkgcache.bin ] && [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] +if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] then echo apt-get update > /dev/null 2>&1 || true fi EOF From bcefeb240c81fd263763c794acc7f6ce0b8f6db0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Feb 2020 15:51:15 +0100 Subject: [PATCH 092/138] add note about IPv6, because people ask --- docs/src/index.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/index.rst b/docs/src/index.rst index 5e54d8fc..31c044dc 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -2,8 +2,9 @@ cdist - usable configuration management ======================================= cdist is a usable configuration management system. -It adheres to the KISS principle and +It adheres to the KISS principle and is being used in small up to enterprise grade environments. +It natively supports IPv6 since the first release. .. toctree:: From 710e99f240ef616c6c1978167749b877b3d1e8dc Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 13 Feb 2020 17:29:44 +0200 Subject: [PATCH 093/138] __mysql_privileges: fix quoting --- cdist/conf/type/__mysql_privileges/explorer/state | 2 +- cdist/conf/type/__mysql_privileges/gencode-remote | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__mysql_privileges/explorer/state b/cdist/conf/type/__mysql_privileges/explorer/state index 0cfbaacd..4f13a70c 100755 --- a/cdist/conf/type/__mysql_privileges/explorer/state +++ b/cdist/conf/type/__mysql_privileges/explorer/state @@ -30,7 +30,7 @@ host="$( cat "$__object/parameter/host" )" check_privileges="$( mysql -B -N -e "show grants for '$user'@'$host'" \ - | grep -Ei "^grant $privileges on .$database.\..$table. to " || true )" + | grep -Ei "^grant $privileges on .$database.\..?$table.? to " || true )" if [ -n "$check_privileges" ] then diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote index bcd362e6..20975c50 100755 --- a/cdist/conf/type/__mysql_privileges/gencode-remote +++ b/cdist/conf/type/__mysql_privileges/gencode-remote @@ -37,13 +37,18 @@ user="$( cat "$__object/parameter/user" )" host="$( cat "$__object/parameter/host" )" +if [ "$table" != '*' ] +then + table="$( printf '`%s`' "$table" )" +fi + case "$state_should" in present) - echo "mysql -e 'grant $privileges on \`$database\`.\`$table\` to \`$user\`@\`$host\`'" + echo "mysql -e 'grant $privileges on \`$database\`.$table to \`$user\`@\`$host\`'" echo "grant $privileges on $database.$table to $user@$host" >> "$__messages_out" ;; absent) - echo "mysql -e 'revoke $privileges on \`$database\`.\`$table\` from \`$user\`@\`$host\`'" + echo "mysql -e 'revoke $privileges on \`$database\`.$table from \`$user\`@\`$host\`'" echo "revoke $privileges on $database.$table from $user@$host" >> "$__messages_out" ;; esac From b7483d73ebc5a95c96adc4a40173ce5015173eeb Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 13 Feb 2020 17:38:19 +0200 Subject: [PATCH 094/138] __mysql_privileges: fix shellcheck --- cdist/conf/type/__mysql_privileges/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__mysql_privileges/gencode-remote b/cdist/conf/type/__mysql_privileges/gencode-remote index 20975c50..0656699f 100755 --- a/cdist/conf/type/__mysql_privileges/gencode-remote +++ b/cdist/conf/type/__mysql_privileges/gencode-remote @@ -39,6 +39,7 @@ host="$( cat "$__object/parameter/host" )" if [ "$table" != '*' ] then + # shellcheck disable=SC2016 table="$( printf '`%s`' "$table" )" fi From f771840178ed2586a4abc47f5e513da2309d5128 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 13 Feb 2020 17:38:34 +0200 Subject: [PATCH 095/138] __mysql_privileges: fix typo --- cdist/conf/type/__mysql_privileges/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__mysql_privileges/man.rst b/cdist/conf/type/__mysql_privileges/man.rst index 8208d7d4..b72c9eba 100644 --- a/cdist/conf/type/__mysql_privileges/man.rst +++ b/cdist/conf/type/__mysql_privileges/man.rst @@ -17,7 +17,7 @@ REQUIRED PARAMETERS database Name of database. -User +user Name of user. From e5f9d320049dea5acef6e505cc7a4966ecbd72c6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 14 Feb 2020 16:07:06 +0100 Subject: [PATCH 096/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bf0a8777..3f69899a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __consul_agent: Add Debian 10 support (Nico Schottelius) * Explorer os_release: Add fallbacks (Dennis Camera) * Types __file, __directory: Add fallback for systems without stat (Dennis Camera) + * Type __mysql_privileges: Fix quoting (Ander Punnar) 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) From 1be5a9d3163fc5b8aa107764ea5aab41f46e860a Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 14 Feb 2020 19:19:12 +0200 Subject: [PATCH 097/138] __package_apt: add note about updating index --- cdist/conf/type/__package_apt/man.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst index a3a70d91..a1691eac 100644 --- a/cdist/conf/type/__package_apt/man.rst +++ b/cdist/conf/type/__package_apt/man.rst @@ -11,6 +11,9 @@ DESCRIPTION apt-get is usually used on Debian and variants (like Ubuntu) to manage packages. +This type will also update package index, if it is older +than one day, to avoid missing package error messages. + REQUIRED PARAMETERS ------------------- From 869a38676f97b7863997b7d2c257447a63344368 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 14 Feb 2020 19:22:06 +0200 Subject: [PATCH 098/138] __package_apt: add comment about package index update --- cdist/conf/type/__package_apt/gencode-remote | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index cb79e886..e02564a2 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -74,6 +74,9 @@ fi case "$state_should" in present) + # following is bit ugly, but important hack. + # due to how cdist config run works, there isn't + # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] then echo apt-get update > /dev/null 2>&1 || true From 48bff6656322eecd940a1e0a2c272fe2303268f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 14 Feb 2020 21:55:35 +0100 Subject: [PATCH 099/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3f69899a..734db7ea 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * Explorer os_release: Add fallbacks (Dennis Camera) * Types __file, __directory: Add fallback for systems without stat (Dennis Camera) * Type __mysql_privileges: Fix quoting (Ander Punnar) + * Type __package_apt: Update package index if it is older than one day (Ander Punnar) 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) From a761d4842ca36688fd6355a63185da985528366a Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 15 Feb 2020 08:21:24 +0100 Subject: [PATCH 100/138] Add check for the state parameter in manifest. This only allow following cases: - present - absent else, it will abort with an error message. --- cdist/conf/type/__cron/manifest | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cdist/conf/type/__cron/manifest b/cdist/conf/type/__cron/manifest index 53973e07..3017e2fd 100755 --- a/cdist/conf/type/__cron/manifest +++ b/cdist/conf/type/__cron/manifest @@ -22,3 +22,12 @@ if [ -f "$__object/parameter/raw" ] && [ -f "$__object/parameter/raw_command" ]; echo "ERROR: both raw and raw_command specified" >&2 exit 1 fi + +case "$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" in + present) ;; + absent) ;; + + *) + echo "ERROR: unkown cron state" >&2 + exit 2 +esac From 5e6e17b3e55ea5b53a3ad9fe28f1d1c422121369 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 15 Feb 2020 09:46:23 +0100 Subject: [PATCH 101/138] Moved default parameter values from scripts to cdist parameter handling. For more generalisation, the default parameter values are now handled by cdist instead of trying to get a value and use a default if parameter is not given. It handles the default values in a more general way, instead of write one default in (possibly) multiple places. Problem occurred when checking the 'state' parameter, which required to set a default value in two places. --- cdist/conf/type/__cron/gencode-remote | 12 ++++++------ cdist/conf/type/__cron/manifest | 2 +- .../conf/type/__cron/parameter/default/day_of_month | 1 + cdist/conf/type/__cron/parameter/default/day_of_week | 1 + cdist/conf/type/__cron/parameter/default/hour | 1 + cdist/conf/type/__cron/parameter/default/minute | 1 + cdist/conf/type/__cron/parameter/default/month | 1 + cdist/conf/type/__cron/parameter/default/state | 1 + 8 files changed, 13 insertions(+), 7 deletions(-) create mode 100644 cdist/conf/type/__cron/parameter/default/day_of_month create mode 100644 cdist/conf/type/__cron/parameter/default/day_of_week create mode 100644 cdist/conf/type/__cron/parameter/default/hour create mode 100644 cdist/conf/type/__cron/parameter/default/minute create mode 100644 cdist/conf/type/__cron/parameter/default/month create mode 100644 cdist/conf/type/__cron/parameter/default/state diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote index 8f3c92dc..9debbc47 100755 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -31,11 +31,11 @@ if [ -f "$__object/parameter/raw" ]; then elif [ -f "$__object/parameter/raw_command" ]; then entry="$command" else - minute="$(cat "$__object/parameter/minute" 2>/dev/null || echo "*")" - hour="$(cat "$__object/parameter/hour" 2>/dev/null || echo "*")" - day_of_month="$(cat "$__object/parameter/day_of_month" 2>/dev/null || echo "*")" - month="$(cat "$__object/parameter/month" 2>/dev/null || echo "*")" - day_of_week="$(cat "$__object/parameter/day_of_week" 2>/dev/null || echo "*")" + minute="$(cat "$__object/parameter/minute")" + hour="$(cat "$__object/parameter/hour")" + day_of_month="$(cat "$__object/parameter/day_of_month")" + month="$(cat "$__object/parameter/month")" + day_of_week="$(cat "$__object/parameter/day_of_week")" entry="$minute $hour $day_of_month $month $day_of_week $command # $name" fi @@ -52,7 +52,7 @@ else state_is=absent fi -state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" +state_should="$(cat "$__object/parameter/state")" [ "$state_is" = "$state_should" ] && exit 0 diff --git a/cdist/conf/type/__cron/manifest b/cdist/conf/type/__cron/manifest index 3017e2fd..e7b51863 100755 --- a/cdist/conf/type/__cron/manifest +++ b/cdist/conf/type/__cron/manifest @@ -23,7 +23,7 @@ if [ -f "$__object/parameter/raw" ] && [ -f "$__object/parameter/raw_command" ]; exit 1 fi -case "$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" in +case "$(cat "$__object/parameter/state")" in present) ;; absent) ;; diff --git a/cdist/conf/type/__cron/parameter/default/day_of_month b/cdist/conf/type/__cron/parameter/default/day_of_month new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__cron/parameter/default/day_of_month @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__cron/parameter/default/day_of_week b/cdist/conf/type/__cron/parameter/default/day_of_week new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__cron/parameter/default/day_of_week @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__cron/parameter/default/hour b/cdist/conf/type/__cron/parameter/default/hour new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__cron/parameter/default/hour @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__cron/parameter/default/minute b/cdist/conf/type/__cron/parameter/default/minute new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__cron/parameter/default/minute @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__cron/parameter/default/month b/cdist/conf/type/__cron/parameter/default/month new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__cron/parameter/default/month @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__cron/parameter/default/state b/cdist/conf/type/__cron/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__cron/parameter/default/state @@ -0,0 +1 @@ +present From ef089d1c6114e840c4b8022d43322bdb18b96e8f Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Mon, 6 Jan 2020 10:42:12 +0100 Subject: [PATCH 102/138] [__systemd_service] new type to manage the state of systemd services --- .../type/__systemd_service/explorer/state | 43 +++++++ .../type/__systemd_service/gencode-remote | 98 ++++++++++++++++ cdist/conf/type/__systemd_service/man.rst | 110 ++++++++++++++++++ .../type/__systemd_service/parameter/boolean | 1 + .../__systemd_service/parameter/default/state | 1 + .../type/__systemd_service/parameter/optional | 3 + 6 files changed, 256 insertions(+) create mode 100755 cdist/conf/type/__systemd_service/explorer/state create mode 100755 cdist/conf/type/__systemd_service/gencode-remote create mode 100644 cdist/conf/type/__systemd_service/man.rst create mode 100644 cdist/conf/type/__systemd_service/parameter/boolean create mode 100644 cdist/conf/type/__systemd_service/parameter/default/state create mode 100644 cdist/conf/type/__systemd_service/parameter/optional diff --git a/cdist/conf/type/__systemd_service/explorer/state b/cdist/conf/type/__systemd_service/explorer/state new file mode 100755 index 00000000..f5f751d4 --- /dev/null +++ b/cdist/conf/type/__systemd_service/explorer/state @@ -0,0 +1,43 @@ +#!/bin/sh -e +# explorer/state +# +# 2020 Matthias Stecher +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# Check if the service is running or stopped. +# +# The explorer must check before if the service exist, because 'systemctl is-active' +# will return "inactive" even if there is no service there: +# systemctl cat foo # does not exist +# systemctl is-active foo # is "inactive" + + +# get name of the service +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + + +# check if the service exist, else exit without output (also if systemd doesn't exist) +# do not exit here with an error code, will be done in the gencode-remote script +systemctl cat "$name" > /dev/null 2>&1 || exit 0 + +# print if the service is running or not +systemctl is-active -q "$name" && printf "running" || printf "stopped" diff --git a/cdist/conf/type/__systemd_service/gencode-remote b/cdist/conf/type/__systemd_service/gencode-remote new file mode 100755 index 00000000..c867ff22 --- /dev/null +++ b/cdist/conf/type/__systemd_service/gencode-remote @@ -0,0 +1,98 @@ +#!/bin/sh -e +# gencode-remote +# +# 2020 Matthias Stecher +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# Checks the given state of the service and set it to the given +# state. Optionally, it executes the action if service running. + + +# get name of the service +name="$__object/parameter/name" +if [ -f "$name" ]; then + name="$(cat "$name")" +else + name="$__object_id" +fi + + +# read current status and parameters +state="$(cat "$__object/explorer/state")" +should="$(cat "$__object/parameter/state")" + +# if systemd/service does not exist +if [ -z "$state" ]; then + printf "systemd or service '%s' does not exist!\n" "$name" >&2 + exit 1 +fi + + +# save the action required +required_action="" + +# check the state of the service that should be +if [ "$state" != "$should" ]; then + # select what to do to get the $should state + case "$should" in + running) + if [ "$state" = "stopped" ]; then required_action="start"; fi + ;; + + stopped) + if [ "$state" = "running" ]; then required_action="stop"; fi + ;; + esac +fi + +# check if the action can be achieved if given +if [ -f "$__object/parameter/action" ] \ + && [ -z "$required_action" ] && [ "$state" = "running" ]; then + + # there must be an action + action="$(cat "$__object/parameter/action")" + + # select the action to the required element + case "$action" in + restart) + required_action="restart" + ;; + + reload) + required_action="reload" + ;; + + *) + printf "action '%s' does not exist!" "$action" >&2 + exit 2 + esac + + # Make a special check: only do this action if a dependency did something + # it is required that the dependencies write there action to $__messages_in + if [ -f "$__object/parameter/if-required" ]; then + # exit here if there are no changes from the dependencies affected (nothing to do) + if ! grep -q -f "$__object/require" "$__messages_in"; then exit 0; fi + fi +fi + +# print the execution command if a action given +if [ -n "$required_action" ]; then + # also print it as message + echo "$required_action" >> "$__messages_out" + echo "systemctl $required_action '$name'" +fi diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst new file mode 100644 index 00000000..678cee29 --- /dev/null +++ b/cdist/conf/type/__systemd_service/man.rst @@ -0,0 +1,110 @@ +cdist-type__systemd-service(7) +============================== + +NAME +---- +cdist-type__systemd-service - Controls a systemd service state + +DESCRIPTION +----------- +This type controls systemd services to define a state of the service, +or an action like reloading or restarting. It is useful to reload a +service after configuration applied or shutdown one service. + +The activation or deactivation is out of scope. Look for the +:strong:`cdist-type__systemd_util`\ (7) type instead. + +REQUIRED PARAMETERS +------------------- + +None. + +OPTIONAL PARAMETERS +------------------- + +name + String which will used as name instead of the object id. + +state + The state which the service should be in: + + running + Service should run (default) + + stoppend + Service should stopped + +action + Executes an action on on the service. It will only execute it if the + service keeps the state **running**. There are following actions, where: + + reload + Reloads the service + + restart + Restarts the service + +BOOLEAN PARAMETERS +----------------- + +if-required + Only execute the action if minimum one required type outputs a message to + **$__messages_out**. Through this, the action should only executed if a + dependency did something. The action will not executed if no dependencies + given. + +MESSAGES +-------- + +start + Started the service + +stop + Stopped the service + +restart + Restarted the service + +reload + Reloaded the service + +ABORTS +------ +Aborts in following cases: + +systemd or the service does not exist + +EXAMPLES +-------- +.. code-block:: sh + + # service must run + __systemd_service nginx + + # service must stopped + __systemd_service sshd \ + --state stopped + + # restart the service + __systemd_service apache2 \ + --action restart + + # makes sure the service exist with an alternative name + __systemd_service foo \ + --name sshd + + # reload the service for a modified configuration file + # only reloads the service if the file really changed + require="__config_file/etc/foo.conf" __systemd_service foo \ + --action reload --if-required + +AUTHORS +------- +Matthias Stecher + +COPYRIGHT +--------- +Copyright \(C) 2020 Matthias Stecher. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__systemd_service/parameter/boolean b/cdist/conf/type/__systemd_service/parameter/boolean new file mode 100644 index 00000000..a4bccb66 --- /dev/null +++ b/cdist/conf/type/__systemd_service/parameter/boolean @@ -0,0 +1 @@ +if-required diff --git a/cdist/conf/type/__systemd_service/parameter/default/state b/cdist/conf/type/__systemd_service/parameter/default/state new file mode 100644 index 00000000..a2ae71b3 --- /dev/null +++ b/cdist/conf/type/__systemd_service/parameter/default/state @@ -0,0 +1 @@ +running diff --git a/cdist/conf/type/__systemd_service/parameter/optional b/cdist/conf/type/__systemd_service/parameter/optional new file mode 100644 index 00000000..fc78265f --- /dev/null +++ b/cdist/conf/type/__systemd_service/parameter/optional @@ -0,0 +1,3 @@ +name +state +action From 58841fc4bfd8dbeb82717fe3638b3bb4d2ed017b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Feb 2020 12:20:41 +0100 Subject: [PATCH 103/138] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 734db7ea..f687e5ed 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,8 @@ next: * Types __file, __directory: Add fallback for systems without stat (Dennis Camera) * Type __mysql_privileges: Fix quoting (Ander Punnar) * Type __package_apt: Update package index if it is older than one day (Ander Punnar) + * Type __cron: Fix job removal if 'is' and 'should' don't match (Matthias Stecher) + * New type: __systemd_service (Matthias Stecher) 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) From a1536933ab066671e09aaf47a3c6101a68a6b60f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 15 Feb 2020 08:51:29 +0100 Subject: [PATCH 104/138] Fix password command synthax in __postgres_role --- cdist/conf/type/__postgres_role/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 977832c9..282294c9 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -54,7 +54,7 @@ case "$state_should" in [ -n "$password" ] && password="PASSWORD '$password'" cat << EOF -su - '$postgres_user' -c "psql postgres -wc 'CREATE ROLE \"$name\" WITH $password $booleans;'" +su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\"" EOF ;; absent) From cda77e5e8a44eabde7e39396c39fd34daba1c702 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Feb 2020 14:35:29 +0100 Subject: [PATCH 105/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f687e5ed..d5ed1b06 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __package_apt: Update package index if it is older than one day (Ander Punnar) * Type __cron: Fix job removal if 'is' and 'should' don't match (Matthias Stecher) * New type: __systemd_service (Matthias Stecher) + * Type __postgres_role: Fix password command synthax (Timothée Floure) 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) From 9f3747cf3f664ba5cc2f74ec356d833e7eda2621 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Feb 2020 14:36:26 +0100 Subject: [PATCH 106/138] Release 6.5.1 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index d5ed1b06..9f1e6245 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) * Explorer os_release: Add fallbacks (Dennis Camera) * Types __file, __directory: Add fallback for systems without stat (Dennis Camera) From 0640b02f90d92e11f6a4f8b7c0110d9b5a9bfb89 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Feb 2020 14:59:12 +0100 Subject: [PATCH 107/138] Fix too short title underline --- cdist/conf/type/__systemd_service/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__systemd_service/man.rst b/cdist/conf/type/__systemd_service/man.rst index 678cee29..7eca398b 100644 --- a/cdist/conf/type/__systemd_service/man.rst +++ b/cdist/conf/type/__systemd_service/man.rst @@ -45,7 +45,7 @@ action Restarts the service BOOLEAN PARAMETERS ------------------ +------------------ if-required Only execute the action if minimum one required type outputs a message to From c09165d1228e887e2fb4dad456c6b1409211a499 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Feb 2020 21:05:24 +0100 Subject: [PATCH 108/138] Fix spelling --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 9f1e6245..bd767b9d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,7 +9,7 @@ Changelog * Type __package_apt: Update package index if it is older than one day (Ander Punnar) * Type __cron: Fix job removal if 'is' and 'should' don't match (Matthias Stecher) * New type: __systemd_service (Matthias Stecher) - * Type __postgres_role: Fix password command synthax (Timothée Floure) + * Type __postgres_role: Fix password command syntax (Timothée Floure) 6.5.0: 2020-01-23 * Type __acl: Add --entry parameter to replace --acl, deprecate --acl (Ander Punnar) From 70200cd28fe9b35670f574e77d49e747f23072f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jan 2020 10:06:09 +0100 Subject: [PATCH 109/138] Refactor __consul_agent type to support distribution packages --- cdist/conf/type/__consul_agent/man.rst | 3 + cdist/conf/type/__consul_agent/manifest | 373 +++++++++++------- .../type/__consul_agent/parameter/boolean | 1 + 3 files changed, 225 insertions(+), 152 deletions(-) diff --git a/cdist/conf/type/__consul_agent/man.rst b/cdist/conf/type/__consul_agent/man.rst index 966abc60..62ee70bb 100644 --- a/cdist/conf/type/__consul_agent/man.rst +++ b/cdist/conf/type/__consul_agent/man.rst @@ -116,6 +116,9 @@ verify-incoming verify-outgoing enforce the use of TLS and verify the peers authenticity on outgoing connections +use-distribution-package + uses distribution package instead of upstream binary + EXAMPLES -------- diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index e00f29ec..599f15b4 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -2,6 +2,7 @@ # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # 2015-2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2019 Timothée Floure (timothee.floure at ungleich.ch) # # This file is part of cdist. # @@ -19,133 +20,64 @@ # along with cdist. If not, see . # - os=$(cat "$__global/explorer/os") -case "$os" in - alpine|scientific|centos|debian|devuan|redhat|ubuntu) - # whitelist safeguard - : - ;; - *) - echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 - echo "Please contribute an implementation for it if you can." >&2 - exit 1 - ;; -esac +### +# Type parameters. state="$(cat "$__object/parameter/state")" user="$(cat "$__object/parameter/user")" group="$(cat "$__object/parameter/group")" +release=$(cat "$__global/explorer/lsb_release") +if [ -f "$__object/parameter/use-distribution-package" ]; then + use_distribution_package=1 +fi + +### +# Those are default that might be overriden by os-specific logic. + data_dir="/var/lib/consul" conf_dir="/etc/consul/conf.d" conf_file="config.json" +tls_dir="$conf_dir/tls" -# FIXME: there has got to be a better way to handle the dependencies in this case -case "$state" in - present) - __group "$group" --system --state "$state" - require="__group/$group" \ - __user "$user" --system --gid "$group" \ - --home "$data_dir" --state "$state" - export require="__user/consul" - ;; - absent) - echo "Sorry, state=absent currently not supported :-(" >&2 - exit 1 - require="$__object_name" \ - __user "$user" --system --gid "$group" --state "$state" - require="__user/$user" \ - __group "$group" --system --state "$state" - ;; -esac +### +# Sane deployment, based on distribution package when available. -__directory /etc/consul \ - --owner root --group "$group" --mode 750 --state "$state" -require="__directory/etc/consul" \ - __directory "$conf_dir" \ - --owner root --group "$group" --mode 750 --state "$state" +distribution_setup () { + case "$os" in + debian) + # consul is only available starting Debian 10 (buster). + # See https://packages.debian.org/buster/consul + if [ $release -lt 10 ]; then + echo "Consul is not available for your debian release." >&2 + echo "Please use the 'manual' (i.e. non-package) installation or \ + upgrade the target system." >&2 + exit 1 + fi -if [ -f "$__object/parameter/ca-file-source" ] || [ -f "$__object/parameter/cert-file-source" ] || [ -f "$__object/parameter/key-file-source" ]; then - # create directory for ssl certs - require="__directory/etc/consul" \ - __directory /etc/consul/ssl \ - --owner root --group "$group" --mode 750 --state "$state" -fi + # Override previously defined environment to match debian packaging. + conf_dir='/etc/consul.d' + user='consul' + grou='consul' + ;; + *) + echo "Your operating system ($os) is currently not supported with the \ + --use-distribution-package flag (${__type##*/})." >&2 + echo "Please use non-package installation or contribute an \ + implementation for if you can." >&2 + exit 1 + ;; + esac -__directory "$data_dir" \ - --owner "$user" --group "$group" --mode 770 --state "$state" + # Install consul package. + __package consul --state $state + export config_deployment_requires="__package/consul" +} -# Generate json config file -( -echo "{" - -# parameters we define ourself -printf ' "data_dir": "%s"\n' "$data_dir" - -cd "$__object/parameter/" -for param in *; do - case "$param" in - state|user|group|json-config) continue ;; - ca-file-source|cert-file-source|key-file-source) - source="$(cat "$__object/parameter/$param")" - destination="/etc/consul/ssl/${source##*/}" - require="__directory/etc/consul/ssl" \ - __file "$destination" \ - --owner root --group consul --mode 640 \ - --source "$source" \ - --state "$state" - key="$(echo "${param%-*}" | tr '-' '_')" - printf ' ,"%s": "%s"\n' "$key" "$destination" - ;; - disable-remote-exec|disable-update-check|leave-on-terminate|rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing) - # handle boolean parameters - key="$(echo "$param" | tr '-' '_')" - printf ' ,"%s": true\n' "$key" - ;; - retry-join) - # join multiple parameters into json array - retry_join="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join")" - # remove trailing , - printf ' ,"retry_join": [%s]\n' "${retry_join%*,}" - ;; - retry-join-wan) - # join multiple parameters into json array over wan - retry_join_wan="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join-wan")" - # remove trailing , - printf ' ,"retry_join_wan": [%s]\n' "${retry_join_wan%*,}" - ;; - bootstrap-expect) - # integer key=value parameters - key="$(echo "$param" | tr '-' '_')" - printf ' ,"%s": %s\n' "$key" "$(cat "$__object/parameter/$param")" - ;; - *) - # string key=value parameters - key="$(echo "$param" | tr '-' '_')" - printf ' ,"%s": "%s"\n' "$key" "$(cat "$__object/parameter/$param")" - ;; - esac -done -if [ -f "$__object/parameter/json-config" ]; then - json_config="$(cat "$__object/parameter/json-config")" - if [ "$json_config" = "-" ]; then - json_config="$__object/stdin" - fi - # remove leading and trailing whitespace and commas from first and last line - # indent each line with 3 spaces for consistency - json=$(sed -e 's/^[ \t]*/ /' -e '1s/^[ \t,]*//' -e '$s/[ \t,]*$//' "$json_config") - printf ' ,%s\n' "$json" -fi -echo "}" -) | \ -require="__directory${conf_dir}" \ - __config_file "${conf_dir}/${conf_file}" \ - --owner root --group "$group" --mode 640 \ - --state "$state" \ - --onchange 'service consul status >/dev/null && service consul reload || true' \ - --source - +### +# LEGACY manual deployment, kept for compatibility reasons. init_sysvinit() { @@ -179,47 +111,184 @@ init_upstart() require="__file/etc/init/consul.conf" __start_on_boot consul } -# Install init script to start on boot -case "$os" in - devuan) - init_sysvinit debian - ;; - centos|redhat) - os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")" - major_version="${os_version%%.*}" - case "$major_version" in - [456]) - init_sysvinit redhat - ;; - 7) - init_systemd - ;; - *) - echo "Unsupported CentOS/Redhat version: $os_version" >&2 - exit 1 - ;; - esac - ;; +manual_setup () { + case "$os" in + alpine|scientific|centos|debian|devuan|redhat|ubuntu) + # whitelist safeguard + : + ;; + *) + echo "Your operating system ($os) is currently not supported by this \ + type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; + esac - debian) - os_version=$(cat "$__global/explorer/os_version") - major_version="${os_version%%.*}" + # FIXME: there has got to be a better way to handle the dependencies in this case + case "$state" in + present) + __group "$group" --system --state "$state" + require="__group/$group" __user "$user" \ + --system --gid "$group" --home "$data_dir" --state "$state" + ;; + *) + echo "The $state state is not (yet?) supported by this type." >&2 + exit 1 + ;; + esac - case "$major_version" in - [567]) - init_sysvinit debian - ;; - [89]|10) - init_systemd - ;; - *) - echo "Unsupported Debian version $os_version" >&2 - exit 1 - ;; - esac - ;; + # Create data directory. + require="__user/consul"__directory "$data_dir" \ + --owner "$user" --group "$group" --mode 770 --state "$state" - ubuntu) - init_upstart + # Create config directory. + require="__user/consul" __directory "$conf_dir" \ + --parents --owner root --group "$group" --mode 750 --state "$state" + + # Install init script to start on boot + case "$os" in + devuan) + init_sysvinit debian + ;; + centos|redhat) + os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")" + major_version="${os_version%%.*}" + case "$major_version" in + [456]) + init_sysvinit redhat + ;; + 7) + init_systemd + ;; + *) + echo "Unsupported CentOS/Redhat version: $os_version" >&2 + exit 1 + ;; + esac + ;; + + debian) + os_version=$(cat "$__global/explorer/os_version") + major_version="${os_version%%.*}" + + case "$major_version" in + [567]) + init_sysvinit debian + ;; + [89]|10) + init_systemd + ;; + *) + echo "Unsupported Debian version $os_version" >&2 + exit 1 + ;; + esac + ;; + + ubuntu) + init_upstart + ;; + esac + + config_deployment_requires="__user/consul __directory/$conf_dir" +} + +### +# Trigger requested installation method. +if [ $use_distribution_package ]; then + distribution_setup +else + manual_setup +fi + +### +# Generate and deploy configuration. +json_configuration=$( + echo "{" + + # parameters we define ourself + printf ' "data_dir": "%s"\n' "$data_dir" + + cd "$__object/parameter/" + for param in *; do + case "$param" in + state|user|group|json-config|use-distribution-package) continue ;; + ca-file-source|cert-file-source|key-file-source) + source="$(cat "$__object/parameter/$param")" + destination="/etc/consul/ssl/${source##*/}" + require="__directory/etc/consul/ssl" \ + __file "$destination" \ + --owner root --group consul --mode 640 \ + --source "$source" \ + --state "$state" + key="$(echo "${param%-*}" | tr '-' '_')" + printf ' ,"%s": "%s"\n' "$key" "$destination" ;; -esac + disable-remote-exec|disable-update-check|leave-on-terminate\ + |rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing) + # handle boolean parameters + key="$(echo "$param" | tr '-' '_')" + printf ' ,"%s": true\n' "$key" + ;; + retry-join) + # join multiple parameters into json array + retry_join="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join")" + # remove trailing , + printf ' ,"retry_join": [%s]\n' "${retry_join%*,}" + ;; + retry-join-wan) + # join multiple parameters into json array over wan + retry_join_wan="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join-wan")" + # remove trailing , + printf ' ,"retry_join_wan": [%s]\n' "${retry_join_wan%*,}" + ;; + bootstrap-expect) + # integer key=value parameters + key="$(echo "$param" | tr '-' '_')" + printf ' ,"%s": %s\n' "$key" "$(cat "$__object/parameter/$param")" + ;; + *) + # string key=value parameters + key="$(echo "$param" | tr '-' '_')" + printf ' ,"%s": "%s"\n' "$key" "$(cat "$__object/parameter/$param")" + ;; + esac + done + if [ -f "$__object/parameter/json-config" ]; then + json_config="$(cat "$__object/parameter/json-config")" + if [ "$json_config" = "-" ]; then + json_config="$__object/stdin" + fi + # remove leading and trailing whitespace and commas from first and last line + # indent each line with 3 spaces for consistency + json=$(sed -e 's/^[ \t]*/ /' -e '1s/^[ \t,]*//' -e '$s/[ \t,]*$//' "$json_config") + printf ' ,%s\n' "$json" + fi + echo "}" +) +echo "$json_configuration" | require="$config_deployment_requires" \ + __file "$conf_dir/$conf_file" \ + --owner root --group "$group" --mode 640 \ + --state "$state" \ + --source - + +# Set configuration deployment as requirement for service restart. +restart_requires="__file/$conf_dir/$conf_file" + +### +# Install TLS certificates. +if [ -f "$__object/parameter/ca-file-source" ] || \ + [ -f "$__object/parameter/cert-file-source" ] || \ + [ -f "$__object/parameter/key-file-source" ]; then + + requires="__file/$conf_dir/$conf_file" __directory $conf_dir/tls \ + --owner root --group "$group" --mode 750 --state "$state" + + # Append to service restart requirements. + restart_requires="$restart_requires __directory/$conf_dir/tls" +fi + +### +# Restart consul agent after everything else. +require="$restart_requires" __service consul --action restart diff --git a/cdist/conf/type/__consul_agent/parameter/boolean b/cdist/conf/type/__consul_agent/parameter/boolean index 91f7f17e..c86853c3 100644 --- a/cdist/conf/type/__consul_agent/parameter/boolean +++ b/cdist/conf/type/__consul_agent/parameter/boolean @@ -6,3 +6,4 @@ server enable-syslog verify-incoming verify-outgoing +use-distribution-package From f595664924b3ae2ad190f9469fdf0be3b38d8a47 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jan 2020 10:07:21 +0100 Subject: [PATCH 110/138] Patch __consul_* to discover remote consul configuration dir --- cdist/conf/type/__consul_check/explorer/conf-dir | 1 + cdist/conf/type/__consul_check/manifest | 2 +- .../conf/type/__consul_service/explorer/conf-dir | 15 +++++++++++++++ cdist/conf/type/__consul_service/manifest | 5 ++--- .../type/__consul_watch_checks/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_checks/manifest | 2 +- .../type/__consul_watch_event/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_event/manifest | 2 +- .../type/__consul_watch_key/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_key/manifest | 2 +- .../__consul_watch_keyprefix/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_keyprefix/manifest | 2 +- .../type/__consul_watch_nodes/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_nodes/manifest | 2 +- .../type/__consul_watch_service/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_service/manifest | 2 +- .../__consul_watch_services/explorer/conf-dir | 1 + cdist/conf/type/__consul_watch_services/manifest | 2 +- 18 files changed, 33 insertions(+), 11 deletions(-) create mode 120000 cdist/conf/type/__consul_check/explorer/conf-dir create mode 100644 cdist/conf/type/__consul_service/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_checks/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_event/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_key/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_nodes/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_service/explorer/conf-dir create mode 120000 cdist/conf/type/__consul_watch_services/explorer/conf-dir diff --git a/cdist/conf/type/__consul_check/explorer/conf-dir b/cdist/conf/type/__consul_check/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_check/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_check/manifest b/cdist/conf/type/__consul_check/manifest index c9f7add9..522aa1a9 100755 --- a/cdist/conf/type/__consul_check/manifest +++ b/cdist/conf/type/__consul_check/manifest @@ -19,7 +19,7 @@ # name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="check_${name}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_service/explorer/conf-dir b/cdist/conf/type/__consul_service/explorer/conf-dir new file mode 100644 index 00000000..6e94f781 --- /dev/null +++ b/cdist/conf/type/__consul_service/explorer/conf-dir @@ -0,0 +1,15 @@ +# Determine the configuration directory used by consul. + +check_dir () { + if [ -d "$1" ]; then + echo -n "$1" + exit + fi +} + +check_dir '/etc/consul/conf.d' +check_dir '/etc/consul.d' +check_dir '/etc/consul' + +echo 'Could not determine consul configuration dir. Exiting.' >&2 +exit 1 diff --git a/cdist/conf/type/__consul_service/manifest b/cdist/conf/type/__consul_service/manifest index 60397db7..d16f18e0 100755 --- a/cdist/conf/type/__consul_service/manifest +++ b/cdist/conf/type/__consul_service/manifest @@ -19,7 +19,7 @@ # name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="service_${name}.json" state="$(cat "$__object/parameter/state")" @@ -45,7 +45,7 @@ printf ' "name": "%s"\n' "$name" cd "$__object/parameter/" for param in *; do case "$param" in - state|name|check-interval) continue ;; + state|name|check-interval|conf-dir) continue ;; check-script) printf ' ,"check": {\n' printf ' "script": "%s"\n' "$(cat "$__object/parameter/check-script")" @@ -86,7 +86,6 @@ echo " }" # end json file echo "}" ) | \ -require="__directory${conf_dir}" \ __config_file "${conf_dir}/${conf_file}" \ --owner root --group consul --mode 640 \ --state "$state" \ diff --git a/cdist/conf/type/__consul_watch_checks/explorer/conf-dir b/cdist/conf/type/__consul_watch_checks/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_checks/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_checks/manifest b/cdist/conf/type/__consul_watch_checks/manifest index 5fdd7a74..4976b25a 100755 --- a/cdist/conf/type/__consul_watch_checks/manifest +++ b/cdist/conf/type/__consul_watch_checks/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_watch_event/explorer/conf-dir b/cdist/conf/type/__consul_watch_event/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_event/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_event/manifest b/cdist/conf/type/__consul_watch_event/manifest index 61934656..b17680c1 100755 --- a/cdist/conf/type/__consul_watch_event/manifest +++ b/cdist/conf/type/__consul_watch_event/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_watch_key/explorer/conf-dir b/cdist/conf/type/__consul_watch_key/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_key/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_key/manifest b/cdist/conf/type/__consul_watch_key/manifest index 61934656..b17680c1 100755 --- a/cdist/conf/type/__consul_watch_key/manifest +++ b/cdist/conf/type/__consul_watch_key/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir b/cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_keyprefix/manifest b/cdist/conf/type/__consul_watch_keyprefix/manifest index 61934656..b17680c1 100755 --- a/cdist/conf/type/__consul_watch_keyprefix/manifest +++ b/cdist/conf/type/__consul_watch_keyprefix/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_watch_nodes/explorer/conf-dir b/cdist/conf/type/__consul_watch_nodes/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_nodes/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_nodes/manifest b/cdist/conf/type/__consul_watch_nodes/manifest index 61934656..b17680c1 100755 --- a/cdist/conf/type/__consul_watch_nodes/manifest +++ b/cdist/conf/type/__consul_watch_nodes/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_watch_service/explorer/conf-dir b/cdist/conf/type/__consul_watch_service/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_service/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_service/manifest b/cdist/conf/type/__consul_watch_service/manifest index db38eb18..e8d18328 100755 --- a/cdist/conf/type/__consul_watch_service/manifest +++ b/cdist/conf/type/__consul_watch_service/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__consul_watch_services/explorer/conf-dir b/cdist/conf/type/__consul_watch_services/explorer/conf-dir new file mode 120000 index 00000000..daa712c3 --- /dev/null +++ b/cdist/conf/type/__consul_watch_services/explorer/conf-dir @@ -0,0 +1 @@ +../../__consul_service/explorer/conf-dir \ No newline at end of file diff --git a/cdist/conf/type/__consul_watch_services/manifest b/cdist/conf/type/__consul_watch_services/manifest index 61934656..b17680c1 100755 --- a/cdist/conf/type/__consul_watch_services/manifest +++ b/cdist/conf/type/__consul_watch_services/manifest @@ -20,7 +20,7 @@ cdist_type="${__type##*/}" watch_type="${cdist_type##*_}" -conf_dir="/etc/consul/conf.d" +conf_dir=$(cat "$__object/explorer/conf-dir") conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" From 31ad1bdaad701720303054d668cf9635987edf1f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 26 Jan 2020 15:13:12 +0100 Subject: [PATCH 111/138] Fix various typos and styling errors in __consul_agent, conf-dir explorer --- cdist/conf/type/__consul_agent/manifest | 40 ++++++++++--------- .../type/__consul_service/explorer/conf-dir | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index 599f15b4..40667002 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -49,7 +49,7 @@ distribution_setup () { debian) # consul is only available starting Debian 10 (buster). # See https://packages.debian.org/buster/consul - if [ $release -lt 10 ]; then + if [ "$release" -lt 10 ]; then echo "Consul is not available for your debian release." >&2 echo "Please use the 'manual' (i.e. non-package) installation or \ upgrade the target system." >&2 @@ -59,7 +59,7 @@ distribution_setup () { # Override previously defined environment to match debian packaging. conf_dir='/etc/consul.d' user='consul' - grou='consul' + group='consul' ;; *) echo "Your operating system ($os) is currently not supported with the \ @@ -71,7 +71,7 @@ distribution_setup () { esac # Install consul package. - __package consul --state $state + __package consul --state "$state" export config_deployment_requires="__package/consul" } @@ -139,7 +139,7 @@ manual_setup () { esac # Create data directory. - require="__user/consul"__directory "$data_dir" \ + require="__user/consul" __directory "$data_dir" \ --owner "$user" --group "$group" --mode 770 --state "$state" # Create config directory. @@ -202,8 +202,23 @@ else manual_setup fi +### +# Install TLS certificates. + +if [ -f "$__object/parameter/ca-file-source" ] || \ + [ -f "$__object/parameter/cert-file-source" ] || \ + [ -f "$__object/parameter/key-file-source" ]; then + + requires="$config_deployment_requires" __directory $tls_dir \ + --owner root --group "$group" --mode 750 --state "$state" + + # Append to service restart requirements. + restart_requires="$restart_requires __directory/$conf_dir/tls" +fi + ### # Generate and deploy configuration. + json_configuration=$( echo "{" @@ -216,8 +231,8 @@ json_configuration=$( state|user|group|json-config|use-distribution-package) continue ;; ca-file-source|cert-file-source|key-file-source) source="$(cat "$__object/parameter/$param")" - destination="/etc/consul/ssl/${source##*/}" - require="__directory/etc/consul/ssl" \ + destination="$tls_dir/${source##*/}" + require="__directory/$tls_dir" \ __file "$destination" \ --owner root --group consul --mode 640 \ --source "$source" \ @@ -276,19 +291,6 @@ echo "$json_configuration" | require="$config_deployment_requires" \ # Set configuration deployment as requirement for service restart. restart_requires="__file/$conf_dir/$conf_file" -### -# Install TLS certificates. -if [ -f "$__object/parameter/ca-file-source" ] || \ - [ -f "$__object/parameter/cert-file-source" ] || \ - [ -f "$__object/parameter/key-file-source" ]; then - - requires="__file/$conf_dir/$conf_file" __directory $conf_dir/tls \ - --owner root --group "$group" --mode 750 --state "$state" - - # Append to service restart requirements. - restart_requires="$restart_requires __directory/$conf_dir/tls" -fi - ### # Restart consul agent after everything else. require="$restart_requires" __service consul --action restart diff --git a/cdist/conf/type/__consul_service/explorer/conf-dir b/cdist/conf/type/__consul_service/explorer/conf-dir index 6e94f781..0fc9ef84 100644 --- a/cdist/conf/type/__consul_service/explorer/conf-dir +++ b/cdist/conf/type/__consul_service/explorer/conf-dir @@ -2,7 +2,7 @@ check_dir () { if [ -d "$1" ]; then - echo -n "$1" + printf '%s' "$1" exit fi } From 09540dc6bd63cb338ed8dc27bf69e2cc3547f02f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 17 Feb 2020 11:18:36 +0100 Subject: [PATCH 112/138] Add simple __service type --- .../type/__service/explorer/service-manager | 8 +++ cdist/conf/type/__service/gencode-remote | 9 ++++ cdist/conf/type/__service/man.rst | 51 +++++++++++++++++++ cdist/conf/type/__service/manifest | 15 ++++++ cdist/conf/type/__service/parameter/required | 1 + 5 files changed, 84 insertions(+) create mode 100755 cdist/conf/type/__service/explorer/service-manager create mode 100755 cdist/conf/type/__service/gencode-remote create mode 100644 cdist/conf/type/__service/man.rst create mode 100644 cdist/conf/type/__service/manifest create mode 100644 cdist/conf/type/__service/parameter/required diff --git a/cdist/conf/type/__service/explorer/service-manager b/cdist/conf/type/__service/explorer/service-manager new file mode 100755 index 00000000..55a873fa --- /dev/null +++ b/cdist/conf/type/__service/explorer/service-manager @@ -0,0 +1,8 @@ +#!/bin/sh + +# Assume systemd if systemctl is in PATH. +if [ "$(command -v systemctl)" ]; then + printf "systemd" +else + printf "unknown" +fi diff --git a/cdist/conf/type/__service/gencode-remote b/cdist/conf/type/__service/gencode-remote new file mode 100755 index 00000000..ac62e05f --- /dev/null +++ b/cdist/conf/type/__service/gencode-remote @@ -0,0 +1,9 @@ +#!/bin/sh + +manager="$(cat "$__object/explorer/service-manager")" +name=$__object_id +action="$(cat "$__object/parameter/action")" + +if [ "$manager" = "unknown" ]; then + echo "service '$name' '$action'" +fi diff --git a/cdist/conf/type/__service/man.rst b/cdist/conf/type/__service/man.rst new file mode 100644 index 00000000..f9b23d5b --- /dev/null +++ b/cdist/conf/type/__service/man.rst @@ -0,0 +1,51 @@ +cdist-type__service(7) +====================== + +NAME +---- +cdist-type__service - Run action on a system service + + +DESCRIPTION +----------- +This type allows you to run an action against a system service. + + +REQUIRED PARAMETERS +------------------- +action + Arbitrary parameter passed as action. Usually 'start', 'stop', 'reload' or 'restart'. + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Restart nginx service. + __service nginx --action restart + + # Stop postfix service. + __service postfix --action stop + + +AUTHORS +------- +Timothée Floure + + +COPYING +------- +Copyright \(C) 2019 Timothée Floure. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__service/manifest b/cdist/conf/type/__service/manifest new file mode 100644 index 00000000..cb5af234 --- /dev/null +++ b/cdist/conf/type/__service/manifest @@ -0,0 +1,15 @@ +#!/bin/sh + +manager="$(cat "$__object/explorer/service-manager")" + +name=$__object_id +action="$(cat "$__object/parameter/action")" + +case "$manager" in + systemd) + __systemd_service "$name" --action "$action" + ;; + *) + # Unknown: handled by `service $NAME $action` in gencode-remote. + ;; +esac diff --git a/cdist/conf/type/__service/parameter/required b/cdist/conf/type/__service/parameter/required new file mode 100644 index 00000000..a9f84d41 --- /dev/null +++ b/cdist/conf/type/__service/parameter/required @@ -0,0 +1 @@ +action From b891bb05d5751fe0f80829d7fe54cd56468d5e51 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 19 Feb 2020 12:58:22 +0200 Subject: [PATCH 113/138] __update_alternatives: add state explorer --- cdist/conf/type/__update_alternatives/explorer/state | 8 ++++++++ cdist/conf/type/__update_alternatives/gencode-remote | 4 ++++ 2 files changed, 12 insertions(+) create mode 100755 cdist/conf/type/__update_alternatives/explorer/state diff --git a/cdist/conf/type/__update_alternatives/explorer/state b/cdist/conf/type/__update_alternatives/explorer/state new file mode 100755 index 00000000..04a78aaa --- /dev/null +++ b/cdist/conf/type/__update_alternatives/explorer/state @@ -0,0 +1,8 @@ +#!/bin/sh -e +path="$(cat "$__object/parameter/path")" +name="$__object_id" +link="$(readlink "/etc/alternatives/$name")" +if [ "$path" = "$link" ] +then echo present +else echo absent +fi diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index 0e7b0d89..b632deb2 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -21,6 +21,10 @@ # Setup alternative - no standard way to create, always set # +if [ "$(cat "$__object/explorer/state")" = 'present' ] +then exit 0 +fi + path="$(cat "$__object/parameter/path")" name="$__object_id" echo "update-alternatives --quiet --set '$name' '$path'" From a3bc8f94075e23d8c0032fc8322aba5eebebd41e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 19 Feb 2020 12:59:14 +0200 Subject: [PATCH 114/138] __update_alternatives: remove expired comment --- cdist/conf/type/__update_alternatives/gencode-remote | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index b632deb2..c0b49814 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -17,9 +17,6 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Setup alternative - no standard way to create, always set -# if [ "$(cat "$__object/explorer/state")" = 'present' ] then exit 0 From 1b0caeda13ddfc3d941332ea912983aba01b4819 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 19 Feb 2020 14:49:35 +0100 Subject: [PATCH 115/138] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index bd767b9d..6410d36d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __update_alternatives: Add state explorer (Ander Punnar) + 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) * Explorer os_release: Add fallbacks (Dennis Camera) From 28d3760e29aaf8cdb30390f7e78c2334d7af8bc4 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 20 Feb 2020 09:40:55 +0100 Subject: [PATCH 116/138] [cdist]Update os_version for alpine --- cdist/conf/explorer/os_version | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 4c41695b..1d54ea60 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -70,4 +70,7 @@ case "$("$__explorer/os")" in ubuntu) lsb_release -sr ;; -esac + alpine) + cat /etc/alpine-release + ;; +esac \ No newline at end of file From ceddbd15a04327489916b61e58fdf2d7a7728427 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 20 Feb 2020 10:15:34 +0100 Subject: [PATCH 117/138] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6410d36d..b0ebf789 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __update_alternatives: Add state explorer (Ander Punnar) + * Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon) 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) From 21c9e3db1852eebb88be5454137ceb23159f7dc0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 3 Feb 2020 22:12:21 +0100 Subject: [PATCH 118/138] [explorer/init] Support more init systems --- cdist/conf/explorer/init | 225 ++++++++++++++++++++++++++++++--------- 1 file changed, 174 insertions(+), 51 deletions(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index ceae2e9f..bf1736cd 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -1,7 +1,8 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Daniel Heule (hda at sfs.biz) # Copyright 2017, Philippe Gregoire +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -23,59 +24,181 @@ # for example at linux this value is "init" or "systemd" in most cases # +set -e + +# Expected values: +# Linux: +# Gentoo: +# sysvinit, openrc-init + +# GNU: +# Debian: +# hurd-init, sysvinit + +# [root@fedora-12 ~]# readlink /proc/1/exe +# /sbin/init (deleted) +# [root@fedora-12 ~]# ls -l /proc/1/exe +# lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted) + +# inspired by https://stackoverflow.com/a/33266819 +shreadlink() ( + CDPATH= + target=$1 fname= targetDir= + + # Resolve potential symlinks until the ultimate target is found. + while : + do + if ! test -e "$target" + then + printf 'ERROR: %s does not exist.\n' "'$target'" >&2 + return 1 + fi + + # Change to target dir; necessary for correct resolution of target path. + cd "$(dirname -- "$target")" + + fname=$(basename -- "$target") # Extract filename. + [ "$fname" = '/' ] && fname='' # !! curiously, `basename /` returns '/' + + [ -L "$fname" ] || break + + # Extract [next] target path, which may be defined + # *relative* to the symlink's own directory. + # Note: We parse `ls -l` output to find the symlink target + # which is the only POSIX-compliant, albeit somewhat fragile, way. + # FIXME: Will break if one of the filenames contain ’ -> ’ + target=$(ls -l "$fname" | sed -e 's/^.* -> //') + done + + # Get canonical dir. path + targetDir=$(pwd -P) + + # Output the ultimate target's canonical path. + # Note that we manually resolve paths ending in /. and /.. to make sure we have a normalized path. + if test "$fname" = '.' + then + printf '%s\n' "${targetDir%/}" + elif test "$fname" = '..' + then + # Caveat: something like /var/.. will resolve to /private (assuming /var@ -> /private/var), i.e. the '..' is applied + # AFTER canonicalization. + printf '%s\n' "$(dirname -- "${targetDir}")" + else + printf '%s/%s\n' "${targetDir%/}" "$fname" + fi +) + + case $(uname -s) in - Linux) - if test -d /proc/1/ - then - comm_name=$(cat /proc/1/comm) - else - # BusyBox's versions of ps and pgrep do not support some options - # depending on which compile-time options have been used. - # Both pgrep and ps are tried to get the command name - comm_name=$( - pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }' - || ps -o comm= -p 1 2>/dev/null) - fi + Linux|GNU) + # if test -f /proc/1/comm + # then + # comm_name=$(cat /proc/1/comm) + # else + # BusyBox's versions of ps and pgrep do not support some options + # depending on which compile-time options have been used. + # Both pgrep and ps are tried to get the command name + # comm_name=$( + # pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }' + # || ps -o comm= -p 1 2>/dev/null) + # fi - case $comm_name - in - systemd) - echo systemd - ;; - init) - # It could be anything... + init_exe=$(shreadlink /proc/1/exe) - if test -h /proc/1/exe - then - init_exe=/proc/1/exe - else - init_exe=$(command -v "$comm_name") - fi + if ! test -x "$init_exe" + then + # On some rare occasions it can happen that the + # running init's binary has been replaced. In this + # case Linux adjusts the symlink to "X (deleted)" + case $init_exe + in + *' (deleted)') + init_exe=${init_exe% (deleted)} + test -x "$init_exe" || exit 1 + ;; + *) + exit 1 + ;; + esac + fi - test -x "$comm_exe" || exit 1 + if test "$init_exe" = '/hurd/init' + then + # XXX: Could maybe be removed + echo hurd-init + exit 0 + fi - case $("$comm_exe" --version | head -n 1) - in - *SysV*) - echo init - ;; - *upstart*) - echo upstart - ;; - *) - echo "" - ;; - esac - esac - ;; - FreeBSD|OpenBSD) - ps -o comm= -p 1 2>/dev/null || true - ;; - Darwin) - basename "$(ps -o comm= -p 1 2>/dev/null)" - ;; - *) - # return a empty string as unknown value - echo "" - ;; + comm_name=$(basename "$init_exe") + case $comm_name + in + init) + : # handled below + ;; + systemd) + # NOTE: sd_booted(3) + if test -d /run/systemd/system/ + then + echo systemd + exit 0 + fi + # otherwise: treat like "init" + ;; + *) + echo "$comm_name" + exit 0 + ;; + esac + + # init: it could be anything... + case $("$init_exe" --version 2>/dev/null | head -n 1) + in + SysV*) + # This is a little bit more specific than init + echo sysvinit + exit 0 + ;; + *'GNU Hurd'*) + echo hurd-init + ;; + *upstart*) + echo upstart + exit 0 + ;; + esac + case $("$init_exe" --help 2>/dev/null | head -n 1) + in + BusyBox*) + echo busybox + exit 0 + ;; + esac + + echo init + ;; + FreeBSD|OpenBSD) + ps -o comm= -p 1 2>/dev/null || true + ;; + Darwin) + basename "$(ps -o comm= -p 1 2>/dev/null)" + ;; + SunOS) + comm_name=$(ps -o comm= -p 1 2>/dev/null) + if test "$(basename "$comm_name")" != 'init' + then + echo "${comm_name}" + exit 0 + fi + + # XXX: Is this the correct way?? + if test -f /etc/svc/volatile/svc_nonpersist.db + then + echo smf + exit 0 + fi + ;; + *) + # return a empty string as unknown value + echo "" + ;; esac From d895bb0e87f524e8a64a802e6ff1922e52860ffa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 18 Feb 2020 01:24:41 +0100 Subject: [PATCH 119/138] [explorer/init] Clean up --- cdist/conf/explorer/init | 399 ++++++++++++++++++++++++--------------- 1 file changed, 246 insertions(+), 153 deletions(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index bf1736cd..2d4f07c1 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -20,185 +20,278 @@ # along with cdist. If not, see . # # -# Returns the process name of pid 1 ( normaly the init system ) -# for example at linux this value is "init" or "systemd" in most cases +# Returns the name of the init system (PID 1) # - -set -e - # Expected values: # Linux: +# Adélie Linux: +# sysvinit+openrc +# Alpine Linux: +# busybox-init+openrc +# ArchLinux: +# systemd, sysvinit +# CRUX: +# sysvinit +# Debian: +# systemd, upstart, sysvinit, openrc, ??? +# Devuan: +# sysvinit, ??? # Gentoo: -# sysvinit, openrc-init - +# sysvinit+openrc, openrc-init, systemd +# OpenBMC: +# systemd +# OpenWrt: +# procd, init?? +# RedHat (RHEL, CentOS, Fedora, RedHat Linux, ...): +# systemd, upstart, sysvinit +# Slackware: +# sysvinit +# SuSE: +# systemd, sysvinit +# Ubuntu: +# systemd, upstart, sysvinit +# # GNU: -# Debian: -# hurd-init, sysvinit +# Debian: +# hurd-init, sysvinit +# +# BSD: +# {Free,Open,Net}BSD: +# init +# +# Mac OS X: +# launchd, init +# +# Solaris/Illumos: +# smf, init + # [root@fedora-12 ~]# readlink /proc/1/exe # /sbin/init (deleted) # [root@fedora-12 ~]# ls -l /proc/1/exe # lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted) -# inspired by https://stackoverflow.com/a/33266819 -shreadlink() ( - CDPATH= - target=$1 fname= targetDir= +set -e +#set -x # DEBUG - # Resolve potential symlinks until the ultimate target is found. - while : - do - if ! test -e "$target" - then - printf 'ERROR: %s does not exist.\n' "'$target'" >&2 - return 1 - fi - - # Change to target dir; necessary for correct resolution of target path. - cd "$(dirname -- "$target")" - - fname=$(basename -- "$target") # Extract filename. - [ "$fname" = '/' ] && fname='' # !! curiously, `basename /` returns '/' - - [ -L "$fname" ] || break - - # Extract [next] target path, which may be defined - # *relative* to the symlink's own directory. - # Note: We parse `ls -l` output to find the symlink target - # which is the only POSIX-compliant, albeit somewhat fragile, way. - # FIXME: Will break if one of the filenames contain ’ -> ’ - target=$(ls -l "$fname" | sed -e 's/^.* -> //') - done - - # Get canonical dir. path - targetDir=$(pwd -P) - - # Output the ultimate target's canonical path. - # Note that we manually resolve paths ending in /. and /.. to make sure we have a normalized path. - if test "$fname" = '.' +validate_busybox_init() { + # It is quite common to use SysVinit to stack other init systemd + # (like OpenRC) on top of it. So we check for that, too. + if stacked=$(validate_openrc) then - printf '%s\n' "${targetDir%/}" - elif test "$fname" = '..' - then - # Caveat: something like /var/.. will resolve to /private (assuming /var@ -> /private/var), i.e. the '..' is applied - # AFTER canonicalization. - printf '%s\n' "$(dirname -- "${targetDir}")" + echo "busybox-init+${stacked}" else - printf '%s/%s\n' "${targetDir%/}" "$fname" + echo busybox-init fi -) +} +validate_hurd_init() { + # FIXME: Test me! + test -x /hurd/init || return 1 + grep -q 'GNU Hurd' /hurd/init || return 1 + echo hurd-init +} -case $(uname -s) in - Linux|GNU) - # if test -f /proc/1/comm - # then - # comm_name=$(cat /proc/1/comm) - # else - # BusyBox's versions of ps and pgrep do not support some options - # depending on which compile-time options have been used. - # Both pgrep and ps are tried to get the command name - # comm_name=$( - # pgrep -P0 -l 2>/dev/null | awk '/^1[ \t]/ { print $2 }' - # || ps -o comm= -p 1 2>/dev/null) - # fi +validate_openrc() { + test -f /run/openrc/softlevel || return 1 + echo openrc +} - init_exe=$(shreadlink /proc/1/exe) +validate_procd() { + grep -q 'procd' /sbin/procd || return 1 + echo procd +} - if ! test -x "$init_exe" - then - # On some rare occasions it can happen that the - # running init's binary has been replaced. In this - # case Linux adjusts the symlink to "X (deleted)" - case $init_exe - in - *' (deleted)') - init_exe=${init_exe% (deleted)} - test -x "$init_exe" || exit 1 - ;; - *) - exit 1 - ;; - esac - fi +validate_runit() { + test -d /run/runit || return 1 + echo runit +} - if test "$init_exe" = '/hurd/init' - then - # XXX: Could maybe be removed - echo hurd-init - exit 0 - fi +validate_smf() { + # XXX: Is this the correct way?? + test -f /etc/svc/volatile/svc_nonpersist.db || return 1 + echo smf +} - comm_name=$(basename "$init_exe") - case $comm_name +validate_systemd() { + # NOTE: sd_booted(3) + test -d /run/systemd/system/ || return 1 + # systemctl --version | sed -e '/^systemd/!d;s/^systemd //' + echo systemd +} + +validate_sysvinit() { + test -x /sbin/init \ + && grep -q 'INIT_VERSION=sysvinit-[0-9.]*' /sbin/init \ + || return 1 + + # It is quite common to use SysVinit to stack other init systemd + # (like OpenRC) on top of it. So we check for that, too. + if stacked=$(validate_openrc) + then + echo "sysvinit+${stacked}" + else + echo sysvinit + fi + unset stacked +} + +validate_upstart() { + test -x "$(command -v initctl)" || return 1 + case $(initctl version) + in + *'(upstart '*')') + # if type -d /etc/init + # then + # # modern (DBus-based?) upstart >= 0.5 + # : + # elif type -d /etc/events.d + # then + # # ancient upstart + # : + # fi + echo upstart + ;; + *) + return 1 + ;; + esac +} + +find_init_procfs() ( + # First, check if the required file in procfs exists... + test -h /proc/1/exe || return 1 + + # Find init executable + init_exe=$(ls -l /proc/1/exe 2>/dev/null) + init_exe=${init_exe#* -> } + + if ! test -x "$init_exe" + then + # On some rare occasions it can happen that the + # running init's binary has been replaced. In this + # case Linux adjusts the symlink to "X (deleted)" + case $init_exe in - init) - : # handled below - ;; - systemd) - # NOTE: sd_booted(3) - if test -d /run/systemd/system/ - then - echo systemd - exit 0 - fi - # otherwise: treat like "init" + *' (deleted)') + init_exe=${init_exe% (deleted)} + test -x "$init_exe" || exit 1 ;; *) - echo "$comm_name" - exit 0 + exit 1 ;; esac + fi - # init: it could be anything... - case $("$init_exe" --version 2>/dev/null | head -n 1) - in - SysV*) - # This is a little bit more specific than init - echo sysvinit - exit 0 - ;; - *'GNU Hurd'*) - echo hurd-init - ;; - *upstart*) - echo upstart - exit 0 - ;; - esac - case $("$init_exe" --help 2>/dev/null | head -n 1) - in - BusyBox*) - echo busybox - exit 0 - ;; - esac + echo "${init_exe}" +) - echo init - ;; - FreeBSD|OpenBSD) - ps -o comm= -p 1 2>/dev/null || true - ;; - Darwin) - basename "$(ps -o comm= -p 1 2>/dev/null)" - ;; - SunOS) - comm_name=$(ps -o comm= -p 1 2>/dev/null) - if test "$(basename "$comm_name")" != 'init' - then - echo "${comm_name}" - exit 0 - fi +# BusyBox's versions of ps and pgrep do not support some options +# depending on which compile-time options have been used. - # XXX: Is this the correct way?? - if test -f /etc/svc/volatile/svc_nonpersist.db - then - echo smf - exit 0 - fi - ;; - *) - # return a empty string as unknown value - echo "" - ;; -esac +find_init_pgrep() { + pgrep -P0 -fl 2>/dev/null | awk -F '[[:blank:]]' '$1 == 1 { print $2 }' +} + +find_init_ps() { + case $(uname -s) + in + Darwin|NetBSD) + ps -o ucomm= -p 1 2>/dev/null + ;; + FreeBSD) + ps -o command= -p 1 2>/dev/null | cut -d ' ' -f 1 + ;; + OpenBSD) + ps -o command -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1 + ;; + *) + ps -o comm= -p 1 2>/dev/null + ;; + esac +} + +find_init() { + case $(uname -s) + in + Linux|GNU|NetBSD) + find_init_procfs || find_init_pgrep || find_init_ps + ;; + FreeBSD) + find_init_procfs || find_init_ps + ;; + OpenBSD) + find_init_pgrep || find_init_ps + ;; + Darwin|FreeBSD|SunOS) + find_init_ps + ;; + *) + echo "Don't know how to determine init." >&2 + echo 'Please send a patch.' >&2 + exit 1 + esac +} + +validate_by_comm_name() { + case $1 + in + busybox) + validate_busybox_init + ;; + init) + # FIXME: Do some more magic here! + echo init + ;; + openrc-init) + validate_openrc >/dev/null && echo openrc-init + ;; + runit) + validate_runit + ;; + systemd) + validate_systemd + ;; + *) + # Run validate function by comm name if available. + # Fall back to comm name if either it does not exist or + # returns non-zero. + type "validate_$1" >/dev/null && "validate_$1" || echo $1 + esac +} + +try_all() { + # init: it could be anything... + # We try some approaches to gather more information about init without + # calling it! On some init systemd this triggers a reinitialisation of + # the system which we don't want (e.g. embedded systems). + + validate_sysvinit || \ + validate_openrc || \ + validate_runit || \ + validate_smf || \ + validate_upstart || \ + validate_hurd_init || \ + echo init # fallback +} + +init=$(find_init) + +if test -x "${init}" +then + case $init + in + /hurd/init) + # FIXME: Create validate function + echo hurd-init + ;; + */init) + try_all + ;; + *) + validate_by_comm_name "$(basename "${init}")" + ;; + esac +else + validate_by_comm_name "${init}" +fi From 364340c8d5a1bcb6492d6c8cdfbbd808b13b5024 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 20 Feb 2020 21:34:21 +0100 Subject: [PATCH 120/138] [explorer/init] Refactor and testing --- cdist/conf/explorer/init | 392 ++++++++++++++++++++++++++------------- 1 file changed, 267 insertions(+), 125 deletions(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index 2d4f07c1..db417a14 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -21,7 +21,7 @@ # # # Returns the name of the init system (PID 1) -# + # Expected values: # Linux: # Adélie Linux: @@ -35,122 +35,221 @@ # Debian: # systemd, upstart, sysvinit, openrc, ??? # Devuan: -# sysvinit, ??? +# sysvinit, sysvinit+openrc # Gentoo: # sysvinit+openrc, openrc-init, systemd # OpenBMC: # systemd # OpenWrt: -# procd, init?? +# procd, init??? # RedHat (RHEL, CentOS, Fedora, RedHat Linux, ...): -# systemd, upstart, sysvinit +# systemd, upstart, upstart-legacy, sysvinit # Slackware: # sysvinit # SuSE: # systemd, sysvinit # Ubuntu: -# systemd, upstart, sysvinit +# systemd, upstart, upstart-legacy, sysvinit +# VoidLinux: +# runit # # GNU: # Debian: -# hurd-init, sysvinit +# sysvinit, hurd-init # # BSD: # {Free,Open,Net}BSD: # init # # Mac OS X: -# launchd, init +# launchd, init+SystemStarter # # Solaris/Illumos: -# smf, init +# smf, init??? +# NOTE: init systems can be stacked. This is popular to run OpenRC on top of +# sysvinit (Gentoo) or busybox-init (Alpine), but can also be used to run runit +# as a systemd service. This makes init system detection very complicated +# (which result is expected?) This script tries to untangle some combinations, +# OpenRC on top of sysv or busybox (X+openrc), but will ignore others (runit as +# a systemd service) + +# NOTE: When we have no idea, nothing will be printed! + +# NOTE: +# When trying to gather information about the init system make sure to do so +# without calling the binary! On some systems this triggers a reinitialisation +# of the system which we don't want (e.g. embedded systems). -# [root@fedora-12 ~]# readlink /proc/1/exe -# /sbin/init (deleted) -# [root@fedora-12 ~]# ls -l /proc/1/exe -# lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted) set -e -#set -x # DEBUG -validate_busybox_init() { - # It is quite common to use SysVinit to stack other init systemd +KERNEL_NAME=$(uname -s) + +KNOWN_INIT_SYSTEMS=$(cat </dev/null 2>&1 || return 1 + launchctl getenv PATH >/dev/null || return 1 + echo launchd +} + +check_openrc() { test -f /run/openrc/softlevel || return 1 echo openrc } -validate_procd() { - grep -q 'procd' /sbin/procd || return 1 +check_procd() ( + procd_path=${1:-/sbin/procd} + test -x "${procd_path}" || return 1 + grep -q 'procd' "${procd_path}" || return 1 echo procd -} +) -validate_runit() { +check_runit() { test -d /run/runit || return 1 echo runit } -validate_smf() { +check_smf() { # XXX: Is this the correct way?? test -f /etc/svc/volatile/svc_nonpersist.db || return 1 echo smf } -validate_systemd() { +check_systemd() { # NOTE: sd_booted(3) test -d /run/systemd/system/ || return 1 # systemctl --version | sed -e '/^systemd/!d;s/^systemd //' echo systemd } -validate_sysvinit() { - test -x /sbin/init \ - && grep -q 'INIT_VERSION=sysvinit-[0-9.]*' /sbin/init \ - || return 1 +check_systemstarter() { + test -d /System/Library/StartupItems/ || return 1 + test -f /System/Library/StartupItems/LoginWindow/StartupParameters.plist || return 1 + echo init+SystemStarter +} + +check_sysvinit() ( + init_path=${1:-/sbin/init} + grep -q 'INIT_VERSION=sysvinit-[0-9.]*' "${init_path}" || return 1 # It is quite common to use SysVinit to stack other init systemd # (like OpenRC) on top of it. So we check for that, too. - if stacked=$(validate_openrc) + if stacked=$(check_openrc) then echo "sysvinit+${stacked}" else echo sysvinit fi unset stacked -} +) -validate_upstart() { +check_upstart() { test -x "$(command -v initctl)" || return 1 case $(initctl version) in *'(upstart '*')') - # if type -d /etc/init - # then - # # modern (DBus-based?) upstart >= 0.5 - # : - # elif type -d /etc/events.d - # then - # # ancient upstart - # : - # fi - echo upstart + if test -d /etc/init + then + # modern (DBus-based?) upstart >= 0.5 + echo upstart + elif test -d /etc/event.d + then + # ancient upstart + echo upstart-legacy + else + # whatever... + echo upstart + fi ;; *) return 1 @@ -163,7 +262,7 @@ find_init_procfs() ( test -h /proc/1/exe || return 1 # Find init executable - init_exe=$(ls -l /proc/1/exe 2>/dev/null) + init_exe=$(ls -l /proc/1/exe 2>/dev/null) || return 1 init_exe=${init_exe#* -> } if ! test -x "$init_exe" @@ -171,21 +270,100 @@ find_init_procfs() ( # On some rare occasions it can happen that the # running init's binary has been replaced. In this # case Linux adjusts the symlink to "X (deleted)" - case $init_exe - in - *' (deleted)') - init_exe=${init_exe% (deleted)} - test -x "$init_exe" || exit 1 - ;; - *) - exit 1 - ;; - esac + + # [root@fedora-12 ~]# readlink /proc/1/exe + # /sbin/init (deleted) + # [root@fedora-12 ~]# ls -l /proc/1/exe + # lrwxrwxrwx. 1 root root 0 2020-01-30 23:00 /proc/1/exe -> /sbin/init (deleted) + + init_exe=${init_exe% (deleted)} + test -x "$init_exe" || return 1 fi echo "${init_exe}" ) +guess_by_path() { + case $1 + in + /bin/busybox) + check_busybox_init "$1" && return + ;; + /lib/systemd/systemd) + check_systemd "$1" && return + ;; + /hurd/init) + check_hurd_init "$1" && return + ;; + /sbin/launchd) + check_launchd "$1" && return + ;; + /usr/bin/runit|/sbin/runit) + check_runit "$1" && return + ;; + /sbin/openrc-init) + if check_openrc "$1" >/dev/null + then + echo openrc-init + return + fi + ;; + /sbin/procd) + check_procd && return + ;; + /sbin/init|*/init) + # init: it could be anything -> (explicit) no match + return 1 + ;; + esac + + # No match + return 1 +} + +guess_by_comm_name() { + case $1 + in + busybox) + check_busybox_init && return + ;; + openrc-init) + if check_openrc >/dev/null + then + echo openrc-init + return 0 + fi + ;; + init) + # init could be anything -> no match + return 1 + ;; + *) + # Run check function by comm name if available. + # Fall back to comm name if either it does not exist or + # returns non-zero. + if type "check_$1" >/dev/null + then + "check_$1" && return + else + echo "$1" ; return 0 + fi + esac + + return 1 +} + +check_list() ( + # List must be a multi-line input on stdin (one name per line) + while read init + do + "check_${init}" || continue + return 0 + done + return 1 +) + + # BusyBox's versions of ps and pgrep do not support some options # depending on which compile-time options have been used. @@ -194,25 +372,31 @@ find_init_pgrep() { } find_init_ps() { - case $(uname -s) + case $KERNEL_NAME in - Darwin|NetBSD) - ps -o ucomm= -p 1 2>/dev/null + Darwin) + ps -o command -p 1 2>/dev/null | tail -n +2 ;; FreeBSD) - ps -o command= -p 1 2>/dev/null | cut -d ' ' -f 1 + ps -o args= -p 1 2>/dev/null | cut -d ' ' -f 1 ;; - OpenBSD) - ps -o command -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1 - ;; - *) + Linux) ps -o comm= -p 1 2>/dev/null ;; - esac + NetBSD) + ps -o comm= -p 1 2>/dev/null + ;; + OpenBSD) + ps -o args -p 1 2>/dev/null | tail -n +2 | cut -d ' ' -f 1 + ;; + *) + ps -o args= -p 1 2>/dev/null + ;; + esac | trim # trim trailing whitespace (some ps like Darwin add it) } find_init() { - case $(uname -s) + case $KERNEL_NAME in Linux|GNU|NetBSD) find_init_procfs || find_init_pgrep || find_init_ps @@ -233,65 +417,23 @@ find_init() { esac } -validate_by_comm_name() { - case $1 - in - busybox) - validate_busybox_init - ;; - init) - # FIXME: Do some more magic here! - echo init - ;; - openrc-init) - validate_openrc >/dev/null && echo openrc-init - ;; - runit) - validate_runit - ;; - systemd) - validate_systemd - ;; - *) - # Run validate function by comm name if available. - # Fall back to comm name if either it does not exist or - # returns non-zero. - type "validate_$1" >/dev/null && "validate_$1" || echo $1 - esac -} - -try_all() { - # init: it could be anything... - # We try some approaches to gather more information about init without - # calling it! On some init systemd this triggers a reinitialisation of - # the system which we don't want (e.g. embedded systems). - - validate_sysvinit || \ - validate_openrc || \ - validate_runit || \ - validate_smf || \ - validate_upstart || \ - validate_hurd_init || \ - echo init # fallback -} +# ----- init=$(find_init) -if test -x "${init}" -then - case $init - in - /hurd/init) - # FIXME: Create validate function - echo hurd-init - ;; - */init) - try_all - ;; - *) - validate_by_comm_name "$(basename "${init}")" - ;; - esac -else - validate_by_comm_name "${init}" -fi +# If we got a path, guess by the path first (fall back to file name if no match) +# else guess by file name directly. +{ + test -x "${init}" \ + && guess_by_path "${init}" \ + || guess_by_comm_name "$(basename "${init}")" +} && exit 0 || true + + +# Guessing based on the file path and name didn’t lead to a definitive result. +# +# We go through all of the checks until we find a match. To speed up the +# process, common cases will be checked first based on the underlying kernel. + +{ common_candidates_by_kernel; echo "${KNOWN_INIT_SYSTEMS}"; } \ + | unique | check_list From 0d84c91b4047d3da0571d0262b4b5d9a8f9796b9 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 20 Feb 2020 22:55:46 +0100 Subject: [PATCH 121/138] [explorer/init] Fix unique() for Solaris --- cdist/conf/explorer/init | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index db417a14..0f04a0ee 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -135,7 +135,8 @@ trim() { unique() { # Delete duplicate lines (keeping input order) - awk '!x[$0]++' + # NOTE: Solaris AWK breaks without if/print construct. + awk '{ if (!x[$0]++) print }' } From 0d6bc8e8f8166a3f61dd4da4a0e499499d3702c4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 20 Feb 2020 23:29:21 +0100 Subject: [PATCH 122/138] [explorer/init] Make shellcheck happy --- cdist/conf/explorer/init | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index 0f04a0ee..1b921c68 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -310,7 +310,7 @@ guess_by_path() { fi ;; /sbin/procd) - check_procd && return + check_procd "$1" && return ;; /sbin/init|*/init) # init: it could be anything -> (explicit) no match @@ -356,7 +356,7 @@ guess_by_comm_name() { check_list() ( # List must be a multi-line input on stdin (one name per line) - while read init + while read -r init do "check_${init}" || continue return 0 @@ -408,7 +408,7 @@ find_init() { OpenBSD) find_init_pgrep || find_init_ps ;; - Darwin|FreeBSD|SunOS) + Darwin|SunOS) find_init_ps ;; *) @@ -424,6 +424,7 @@ init=$(find_init) # If we got a path, guess by the path first (fall back to file name if no match) # else guess by file name directly. +# shellcheck disable=SC2015 { test -x "${init}" \ && guess_by_path "${init}" \ From 49fc21ec47d665155edac73bdbae4fed9258f382 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Feb 2020 09:32:03 +0100 Subject: [PATCH 123/138] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index b0ebf789..8463bb89 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,9 @@ Changelog next: * Type __update_alternatives: Add state explorer (Ander Punnar) * Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon) + * Explorer init: Rewrite and support more init systems (Dennis Camera) + * New type: __service (Timothée Floure) + * Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure) 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) From c6aba8d189c2efc7c88f9c595acba7ceae5a4e00 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 23 Feb 2020 22:59:41 +0100 Subject: [PATCH 124/138] [explorer/disks] Fix for NetBSD When connecting over SSH and running /bin/sh, the PATH is missing sbin locations. sysctl is located at /sbin/sysctl on NetBSD. --- cdist/conf/explorer/disks | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index 87a6b5c6..08290bc7 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -1,14 +1,20 @@ -#!/bin/sh +#!/bin/sh -e uname_s="$(uname -s)" -case "${uname_s}" in +case $uname_s in FreeBSD) sysctl -n kern.disks ;; - OpenBSD|NetBSD) + OpenBSD) sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs ;; + NetBSD) + PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" + sysctl -n hw.disknames \ + | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' \ + | xargs + ;; Linux) if command -v lsblk > /dev/null then @@ -23,5 +29,3 @@ case "${uname_s}" in printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2 ;; esac - -exit 0 From e6f683b88633fa0722b3787e3daa39db640ce0f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 24 Feb 2020 09:20:49 +0100 Subject: [PATCH 125/138] Add support for alpine (edge) package to __consul_agent --- cdist/conf/type/__consul_agent/manifest | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index 40667002..0d819d45 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -61,6 +61,17 @@ distribution_setup () { user='consul' group='consul' ;; + alpine) + # consul is only available starting Alpine 3.12 (= edge during the 3.11 cycle). + # See https://pkgs.alpinelinux.org/packages?name=consul&branch=edge + + # Override previously defined environment to match alpine packaging. + conf_dir='/etc/consul' + conf_file='server.json' + data_dir='/var/consul' + user='consul' + group='consul' + ;; *) echo "Your operating system ($os) is currently not supported with the \ --use-distribution-package flag (${__type##*/})." >&2 From d3bd2669ec49fb861016e614893dac280ed5fd35 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 23 Feb 2020 23:07:40 +0100 Subject: [PATCH 126/138] [explorer/disks] Support Linux without lsblk (fallback to sysfs) --- cdist/conf/explorer/disks | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index 08290bc7..0fabc95f 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -16,16 +16,33 @@ case $uname_s in | xargs ;; Linux) - if command -v lsblk > /dev/null + # list of major device numbers toexclude: + # ram disks, floppies, cdroms + # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt + ign_majors='1 2 11' + + if command -v lsblk >/dev/null 2>&1 then - # exclude ram disks, floppies and cdroms - # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt - lsblk -e 1,2,11 -dno name | xargs + lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name | xargs + elif test -d /sys/block/ + then + # shellcheck disable=SC2012 + ls -1 /sys/block/ \ + | awk -v ign_majors="$(echo "$ign_majors" | tr ' ' '|')" ' + { + devfile = "/sys/block/" $0 "/dev" + getline devno < devfile + close(devfile) + if (devno !~ "^(" ign_majors "):") print + }' \ + | xargs else - printf "Don't know how to list disks for %s operating system without lsblk, if you can please submit a patch\n" "${uname_s}" >&2 + echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2 + echo 'If you can, please submit a patch.'>&2 fi ;; *) - printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2 + printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2 + printf 'If you can please submit a patch\n' >&2 ;; esac From 1ef126e16f95e822562978abd895a3c036f7d5c4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 23 Feb 2020 23:08:40 +0100 Subject: [PATCH 127/138] [explorer/disks] Move xargs call to the bottom --- cdist/conf/explorer/disks | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index 0fabc95f..ed1afce4 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -7,13 +7,12 @@ case $uname_s in sysctl -n kern.disks ;; OpenBSD) - sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs + sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' ;; NetBSD) PATH="${PATH}:/usr/local/sbin:/usr/sbin:/sbin" sysctl -n hw.disknames \ - | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' \ - | xargs + | awk 'BEGIN { RS = " " } /^[lsw]d[0-9]+/' ;; Linux) # list of major device numbers toexclude: @@ -23,7 +22,7 @@ case $uname_s in if command -v lsblk >/dev/null 2>&1 then - lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name | xargs + lsblk -e "$(echo "$ign_majors" | tr ' ' ',')" -dno name elif test -d /sys/block/ then # shellcheck disable=SC2012 @@ -34,8 +33,7 @@ case $uname_s in getline devno < devfile close(devfile) if (devno !~ "^(" ign_majors "):") print - }' \ - | xargs + }' else echo "Don't know how to list disks on Linux without lsblk and sysfs." >&2 echo 'If you can, please submit a patch.'>&2 @@ -45,4 +43,5 @@ case $uname_s in printf "Don't know how to list disks for %s operating system.\n" "${uname_s}" >&2 printf 'If you can please submit a patch\n' >&2 ;; -esac +esac \ +| xargs From 6db6dc4ac0950579ce13252dcca6d0f61f5533c6 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 23 Feb 2020 23:14:14 +0100 Subject: [PATCH 128/138] [explorer/disks] Add license header --- cdist/conf/explorer/disks | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks index ed1afce4..24540601 100755 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -1,4 +1,24 @@ #!/bin/sh -e +# +# based on previous work by other people, modified by: +# 2020 Dennis Camera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Finds disks of the system (excl. ram disks, floppy, cdrom) uname_s="$(uname -s)" From b2db864eaf95a50e7e0d31be52f86ab1d0bea480 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 26 Feb 2020 13:01:29 +0100 Subject: [PATCH 129/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8463bb89..35590655 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer init: Rewrite and support more init systems (Dennis Camera) * New type: __service (Timothée Floure) * Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure) + * Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera) 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) From 6b4b6534a1187e2533f627ff485873aeac7c4a32 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 26 Feb 2020 16:12:21 +0100 Subject: [PATCH 130/138] [__directory] Give more precise error message when --state pre-exists --- cdist/conf/type/__directory/gencode-remote | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index e1ab69d7..a1a32ea2 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -109,10 +109,24 @@ case "$state_should" in done ;; pre-exists) - if [ "$type" != "directory" ]; then - echo "Directory \"$destination\" does not exist" >&2 - exit 1 - fi + case $type in + directory) + # all good + exit 0 + ;; + none) + printf 'Directory "%s" does not exist\n' "$destination" >&2 + exit 1 + ;; + file|symlink) + printf 'File "%s" exists and is a %s, but should be a directory\n' "$destination" "$type" >&2 + exit 1 + ;; + *) + printf 'File or directory "%s" is in an unknown state\n' "$destination" >&2 + exit 1 + ;; + esac ;; absent) if [ "$type" = "directory" ]; then From 046f7d0663fb6e117f76747bf71c7976abc90962 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 26 Feb 2020 16:32:03 +0100 Subject: [PATCH 131/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 35590655..4fdfcd68 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * New type: __service (Timothée Floure) * Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure) * Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera) + * Type __directory: Add 'exists' and 'pre-exists' states (Dennis Camera) 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) From 46d2487f08e04d3f6f3cbd434404dda006e83276 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 26 Feb 2020 16:40:53 +0100 Subject: [PATCH 132/138] [__file] Give more precise error message when --state pre-exists --- cdist/conf/type/__file/gencode-local | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index fb9f9a92..231b6927 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -31,12 +31,24 @@ if [ "$state_should" = "pre-exists" ]; then exit 1 fi - if [ "$type" = "file" ]; then - exit 0 # nothing to do - else - echo "File \"$destination\" does not exist" - exit 1 - fi + case $type in + file) + # nothing to do + exit 0 + ;; + none) + printf 'File "%s" does not exist\n' "$destination" >&2 + exit 1 + ;; + directory|symlink) + printf 'File "%s" exists and is a %s, but should be a regular file\n' "$destination" "$type" >&2 + exit 1 + ;; + *) + printf 'File or directory "%s" is in an unknown state\n' "$destination" >&2 + exit 1 + ;; + esac fi upload_file= From 9eacba06bb218bd0627ef633b363495331ac6c5b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 26 Feb 2020 16:48:14 +0100 Subject: [PATCH 133/138] [__file] Treat pre-exists as a special case in gencode-remote as it should never get there --- cdist/conf/type/__file/gencode-remote | 56 ++++++++++++++------------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index b04c471e..815593bd 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -55,37 +55,41 @@ set_owner() { } set_mode() { - echo "chmod '$1' '$destination'" - echo "chmod '$1'" >> "$__messages_out" - fire_onchange=1 + echo "chmod '$1' '$destination'" + echo "chmod '$1'" >> "$__messages_out" + fire_onchange=1 } case "$state_should" in - present|exists|pre-exists) - # Note: Mode - needs to happen last as a chown/chgrp can alter mode by - # clearing S_ISUID and S_ISGID bits (see chown(2)) - for attribute in group owner mode; do - if [ -f "$__object/parameter/$attribute" ]; then - value_should="$(cat "$__object/parameter/$attribute")" + present|exists) + # Note: Mode - needs to happen last as a chown/chgrp can alter mode by + # clearing S_ISUID and S_ISGID bits (see chown(2)) + for attribute in group owner mode; do + if [ -f "$__object/parameter/$attribute" ]; then + value_should="$(cat "$__object/parameter/$attribute")" - # change 0xxx format to xxx format => same as stat returns - if [ "$attribute" = mode ]; then - value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" - fi - - value_is="$(get_current_value "$attribute" "$value_should")" - if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then - "set_$attribute" "$value_should" + # change 0xxx format to xxx format => same as stat returns + if [ "$attribute" = mode ]; then + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" + fi + + value_is="$(get_current_value "$attribute" "$value_should")" + if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then + "set_$attribute" "$value_should" + fi fi + done + if [ -f "$__object/files/set-attributes" ]; then + # set-attributes is created if file is created or uploaded in gencode-local + fire_onchange=1 fi - done - if [ -f "$__object/files/set-attributes" ]; then - # set-attributes is created if file is created or uploaded in gencode-local - fire_onchange=1 - fi - ;; + pre-exists) + # pre-exists should never reach gencode-remote… + exit 1 + ;; + absent) if [ "$type" = "file" ]; then echo "rm -f '$destination'" @@ -101,7 +105,7 @@ case "$state_should" in esac if [ -f "$__object/parameter/onchange" ]; then - if [ -n "$fire_onchange" ]; then - cat "$__object/parameter/onchange" - fi + if [ -n "$fire_onchange" ]; then + cat "$__object/parameter/onchange" + fi fi From da6ccf808ef4417b90e0882abd042f0a40f28a8e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 26 Feb 2020 21:48:08 +0100 Subject: [PATCH 134/138] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 4fdfcd68..9075cf3f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Types __consul_*: Add optional parameter for using distribution packages (Timothée Floure) * Explorer disks: Fix NetBSD, support Linux w/o lsblk (Dennis Camera) * Type __directory: Add 'exists' and 'pre-exists' states (Dennis Camera) + * Type __file: Improve error messages for pre-exists state (Dennis Camera) 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) From 213f1b049c55205ffe233de22e68c4a4738c8b82 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 27 Feb 2020 20:23:04 +0100 Subject: [PATCH 135/138] Release 6.5.2 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 9075cf3f..64b124e8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.5.2: 2020-02-27 * Type __update_alternatives: Add state explorer (Ander Punnar) * Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon) * Explorer init: Rewrite and support more init systems (Dennis Camera) From fb32d6ed3f9421d515b0ec2eabe9ab4fcc6ccacb Mon Sep 17 00:00:00 2001 From: llnu Date: Sun, 8 Mar 2020 16:04:02 +0100 Subject: [PATCH 136/138] alpine uses a different getent lib which doesnt support: getent shadow --- cdist/conf/type/__user/explorer/shadow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow index 73ce0e29..63d38f0d 100755 --- a/cdist/conf/type/__user/explorer/shadow +++ b/cdist/conf/type/__user/explorer/shadow @@ -24,7 +24,7 @@ name=$__object_id case $("$__explorer/os") in - 'freebsd'|'netbsd'|'openbsd') + 'freebsd'|'netbsd'|'openbsd'|'alpine') database='passwd' ;; # Default to using shadow passwords From f7d5f5bc974fff858c9999752badce3c6e8ba72e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 9 Mar 2020 08:02:18 +0100 Subject: [PATCH 137/138] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 64b124e8..93df32a2 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __user: Fix missing shadow for alpine (llnu) + 6.5.2: 2020-02-27 * Type __update_alternatives: Add state explorer (Ander Punnar) * Explorer os_version: Add support for Alpine Linux (Jin-Guk Kwon) From f00e4af5f0f1f49ab93be001468327ea90df6bd0 Mon Sep 17 00:00:00 2001 From: Andrew Schleifer Date: Thu, 26 Mar 2020 21:17:32 +0800 Subject: [PATCH 138/138] fix typo --- cdist/conf/type/__letsencrypt_cert/man.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst index c4ffc6bc..85eb88ea 100644 --- a/cdist/conf/type/__letsencrypt_cert/man.rst +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -59,13 +59,13 @@ MESSAGES -------- change - Certificte was changed. + Certificate was changed. create - Certificte was created. + Certificate was created. remove - Certificte was removed. + Certificate was removed. EXAMPLES --------