From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Sun, 21 Feb 2021 19:59:57 +0000 Subject: [PATCH 01/47] First draft of __apt_pin --- cdist/conf/type/__apt_pin/man.rst | 53 ++++++++++++++++ cdist/conf/type/__apt_pin/manifest | 63 +++++++++++++++++++ .../type/__apt_pin/parameter/default/package | 1 + .../type/__apt_pin/parameter/default/state | 1 + cdist/conf/type/__apt_pin/parameter/optional | 2 + cdist/conf/type/__apt_pin/parameter/required | 2 + 6 files changed, 122 insertions(+) create mode 100644 cdist/conf/type/__apt_pin/man.rst create mode 100755 cdist/conf/type/__apt_pin/manifest create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state create mode 100644 cdist/conf/type/__apt_pin/parameter/optional create mode 100644 cdist/conf/type/__apt_pin/parameter/required diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst new file mode 100644 index 00000000..7fcae6f8 --- /dev/null +++ b/cdist/conf/type/__apt_pin/man.rst @@ -0,0 +1,53 @@ +cdist-type__apt_pin(7) +====================== + +NAME +---- +cdist-type__apt_pin - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __apt_pin + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest new file mode 100755 index 00000000..8dd9770d --- /dev/null +++ b/cdist/conf/type/__apt_pin/manifest @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") +state="$(cat "$__object/parameter/state")" +package="$(cat "$__object/parameter/package")" +distribution="$(cat "$__object/parameter/distribution")" +priority="$(cat "$__object/parameter/priority")" + + +case "$os" in + debian|ubuntu|devuan) + ;; + *) + printf "This type is specific to Debian and it's derivatives" >&2 + printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 + exit 1 + ;; +esac + +if [ "$package" = "*" ]; then + name="default" + +else + name="$__object_id" +fi + +case $distribution in + stabletesting|unsatbel|experimental) + pin="release a=$distribution" + ;; + *) + pin="release n=$distribution" + ;; +esac + + +__file /etc/apt/preferences.d/$name \ + --owner root --group root --mode 0644 \ + --state "$state" \ + --source - << EOF +Package: $package +Pin: $pin +Pin-Priority: $priority +EOF diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/package @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional new file mode 100644 index 00000000..52f01fd2 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -0,0 +1,2 @@ +state +package diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required new file mode 100644 index 00000000..4b4e9741 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -0,0 +1,2 @@ +distribution +priority From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 09:37:36 +0000 Subject: [PATCH 02/47] __apt_pin: Always use $__object_id as preferences.d filename --- cdist/conf/type/__apt_pin/manifest | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 8dd9770d..162b523f 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -36,12 +36,7 @@ case "$os" in ;; esac -if [ "$package" = "*" ]; then - name="default" - -else - name="$__object_id" -fi +name="$__object_id" case $distribution in stabletesting|unsatbel|experimental) From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 11:59:09 +0000 Subject: [PATCH 03/47] Fix shellcheck issues --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 162b523f..b1372ad0 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -48,7 +48,7 @@ case $distribution in esac -__file /etc/apt/preferences.d/$name \ +__file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:17 +0100 Subject: [PATCH 04/47] More sensible defaults; reword debian-only error message --- cdist/conf/type/__apt_pin/manifest | 13 +++++++++---- cdist/conf/type/__apt_pin/nonparallel | 0 cdist/conf/type/__apt_pin/parameter/default/package | 1 - 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 cdist/conf/type/__apt_pin/nonparallel delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index b1372ad0..909bc80d 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -19,9 +19,17 @@ # +name="$__object_id" + os=$(cat "$__global/explorer/os") state="$(cat "$__object/parameter/state")" -package="$(cat "$__object/parameter/package")" + +if [ -f "$__object/parameter/package" ]; then + package="$(cat "$__object/parameter/package")" +else + package=$name +fi + distribution="$(cat "$__object/parameter/distribution")" priority="$(cat "$__object/parameter/priority")" @@ -31,13 +39,10 @@ case "$os" in ;; *) printf "This type is specific to Debian and it's derivatives" >&2 - printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 exit 1 ;; esac -name="$__object_id" - case $distribution in stabletesting|unsatbel|experimental) pin="release a=$distribution" diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package deleted file mode 100644 index 72e8ffc0..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/package +++ /dev/null @@ -1 +0,0 @@ -* From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:33 +0100 Subject: [PATCH 05/47] Add documentation --- cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 7fcae6f8..0c91cdec 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -13,11 +13,21 @@ This space intentionally left blank. REQUIRED PARAMETERS ------------------- -None. +distribution + Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...). OPTIONAL PARAMETERS ------------------- +package + Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + +priority + The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) + +state + Will be passed to underlying `__file` type; see there for valid values and defaults. + None. @@ -31,14 +41,31 @@ EXAMPLES .. code-block:: sh - # TODO - __apt_pin + # Add the bullseye repo to buster, but do not install any pacakges by default + # only if explicitely asked for + __apt_pin bullseye-default \ + --package "*" \ + --distribution bullseye \ + --priority -1 + + require="__apt_pin/bullseye-default" __apt_source bullseye \ + --uri http://deb.debian.org/debian/ \ + --distribution bullseye \ + --component main + # TODO + __apt_pin + + __apt_pin foo --package "foo foo-*" --distribution bullseye + + __foo # Installs the `foo` package internally + + __package foo-plugin-extras SEE ALSO -------- -:strong:`TODO`\ (7) - +:strong:`apt_preferences`\ (7) +:strong:`cdist-type__file`\ (7) AUTHORS ------- From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 28 Jun 2021 12:54:20 +0200 Subject: [PATCH 06/47] Use $__remote_exec and thus the ssh multiplexing --- cdist/conf/type/__rsync/gencode-local | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e36ded2f..36addc36 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,4 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ + -e "$__remote_exec" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 28 Jun 2021 18:09:35 +0100 Subject: [PATCH 07/47] Add quotes to rsync command --- cdist/conf/type/__rsync/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 36addc36..f1bddc16 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,5 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ - -e "$__remote_exec" \ + -e \"$__remote_exec\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 5 Jul 2021 12:38:26 +0100 Subject: [PATCH 08/47] Shell check quoting We're actually echo-ing the command, hence the escape in front of the quotes - the issue Shellcheck alludes too would actually occur, had the escaping bakcslashes been omitted. --- cdist/conf/type/__rsync/gencode-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index f1bddc16..be4feabb 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then done < "$__object/parameter/rsync-opts" fi +# shellcheck disable=SC2086 echo rsync -a \ --no-owner --no-group \ - -e \"$__remote_exec\" \ + -e \"${__remote_exec}\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:28:05 +0200 Subject: [PATCH 09/47] Refine docs even more --- cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 0c91cdec..4229c0cd 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -3,12 +3,12 @@ cdist-type__apt_pin(7) NAME ---- -cdist-type__apt_pin - TODO +cdist-type__apt_pin - Manage apt pinning rules DESCRIPTION ----------- -This space intentionally left blank. +Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.) REQUIRED PARAMETERS @@ -20,7 +20,7 @@ distribution OPTIONAL PARAMETERS ------------------- package - Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) @@ -28,7 +28,6 @@ priority state Will be passed to underlying `__file` type; see there for valid values and defaults. -None. BOOLEAN PARAMETERS @@ -41,8 +40,8 @@ EXAMPLES .. code-block:: sh - # Add the bullseye repo to buster, but do not install any pacakges by default - # only if explicitely asked for + # Add the bullseye repo to buster, but do not install any packages by default, + # only if explicitely asked for (-1 means "never" for apt) __apt_pin bullseye-default \ --package "*" \ --distribution bullseye \ @@ -52,19 +51,19 @@ EXAMPLES --uri http://deb.debian.org/debian/ \ --distribution bullseye \ --component main - # TODO - __apt_pin __apt_pin foo --package "foo foo-*" --distribution bullseye - __foo # Installs the `foo` package internally + __foo # Assuming, this installs the `foo` package internally - __package foo-plugin-extras + __package foo-plugin-extras # Assuming we also need some extra stuff SEE ALSO -------- -:strong:`apt_preferences`\ (7) +:strong:`apt_preferences`\ (5) +:strong:`cdist-type__apt_source`\ (7) +:strong:`cdist-type__apt_backports`\ (7) :strong:`cdist-type__file`\ (7) AUTHORS From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:32:27 +0200 Subject: [PATCH 10/47] Fix typo in distro names... --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 909bc80d..e72a8fdd 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -44,7 +44,7 @@ case "$os" in esac case $distribution in - stabletesting|unsatbel|experimental) + stable|testing|unstable|experimental) pin="release a=$distribution" ;; *) From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Jul 2021 08:09:05 +0200 Subject: [PATCH 11/47] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0c3c64e1..04f826f0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,7 @@ next: * Type __debconf_set_selections: Add state explorer (Dennis Camera) * Core: Implement usable cdist scan (Timothée Floure) * New type: __snakeoil_cert (Ander Punnar) + * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 20:37:02 +0200 Subject: [PATCH 12/47] Release 6.9.7 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 04f826f0..284293c1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 21:02:27 +0200 Subject: [PATCH 13/47] Fix docs code block errors --- cdist/conf/type/__snakeoil_cert/man.rst | 1 + docs/src/cdist-scan.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst index 0b547804..b0b0a2e9 100644 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -38,6 +38,7 @@ cert-path EXAMPLES -------- .. code-block:: sh + __snakeoil_cert localhost-rsa \ --common-name localhost \ --key-type rsa:4096 diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 064e65ff..86b7fab6 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable. Simplest script: .. code-block:: sh + #!/bin/sh case "$1" in @@ -71,6 +72,7 @@ Simplest script: Resolving name from `PTR` DNS record: .. code-block:: sh + #!/bin/sh for cmd in dig sed; do From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 13 Jul 2021 00:13:22 +0300 Subject: [PATCH 14/47] [__rsync] rewrite --- cdist/conf/type/__rsync/gencode-local | 123 +++++++++++++----- cdist/conf/type/__rsync/gencode-remote | 37 ------ cdist/conf/type/__rsync/man.rst | 107 +++++---------- cdist/conf/type/__rsync/manifest | 18 --- .../type/__rsync/parameter/default/options | 1 + cdist/conf/type/__rsync/parameter/optional | 4 +- .../type/__rsync/parameter/optional_multiple | 2 +- 7 files changed, 130 insertions(+), 162 deletions(-) delete mode 100755 cdist/conf/type/__rsync/gencode-remote create mode 100644 cdist/conf/type/__rsync/parameter/default/options diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index be4feabb..612d237e 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,41 +1,100 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -source=$(cat "$__object/parameter/source") -remote_user=$(cat "$__object/parameter/remote-user") +if ! command -v rsync > /dev/null +then + echo 'rsync is missing in local machine' >&2 + exit 1 +fi -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") +src="$( cat "$__object/parameter/source" )" + +if [ ! -e "$src" ] +then + echo "$src not found" >&2 + exit 1 +fi + +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" else - destination="/$__object_id" + dst="/$__object_id" fi -set -- -if [ -f "$__object/parameter/rsync-opts" ]; then - while read -r opts; do - set -- "$@" "--$opts" - done < "$__object/parameter/rsync-opts" +# if source is directory, then make sure that +# source and destination are ending with slash, +# because this is what you almost always want when +# rsyncing two directories. + +if [ -d "$src" ] +then + if ! echo "$src" | grep -Eq '/$' + then + src="$src/" + fi + + if ! echo "$dst" | grep -Eq '/$' + then + dst="$dst/" + fi fi +remote_user="$( cat "$__object/parameter/remote-user" )" + +options="$( cat "$__object/parameter/options" )" + +if [ -f "$__object/parameter/option" ] +then + while read -r l + do + # there's a limitation in argparse: value can't begin with '-'. + # to workaround this, let's prefix opts with '\' in manifest and remove here. + # read more about argparse issue: https://bugs.python.org/issue9334 + + options="$options $( echo "$l" | sed 's/\\//g' )" + done \ + < "$__object/parameter/option" +fi + +if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] +then + options="$options --chown=" + + if [ -f "$__object/parameter/owner" ] + then + owner="$( cat "$__object/parameter/owner" )" + options="$options$owner" + fi + + if [ -f "$__object/parameter/group" ] + then + group="$( cat "$__object/parameter/group" )" + options="$options:$group" + fi +fi + +if [ -f "$__object/parameter/mode" ] +then + mode="$( cat "$__object/parameter/mode" )" + options="$options --chmod=$mode" +fi + +# IMPORTANT +# +# 1. we first dry-run rsync with change summary to find out +# if there are any changes and code generation is needed. +# 2. normally, to get current state or target host, we run +# such operations in type explorers, but that's not +# possible due to how rsync works. +# 3. redirecting output of dry-run to stderr to ease debugging. +# 4. to understand how that cryptic regex works, please +# open rsync manpage and read about --itemize-changes. + # shellcheck disable=SC2086 -echo rsync -a \ - --no-owner --no-group \ - -e \"${__remote_exec}\" \ - -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" +if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ + | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 +then + exit 0 +fi + +echo "rsync $options $src $remote_user@$__target_host:$dst" diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote deleted file mode 100755 index 074246af..00000000 --- a/cdist/conf/type/__rsync/gencode-remote +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") -else - destination="/$__object_id" -fi - -ownergroup="" -if [ -f "$__object/parameter/owner" ]; then - ownergroup=$(cat "$__object/parameter/owner") -fi -if [ -f "$__object/parameter/group" ]; then - ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" -fi - -if [ "$ownergroup" ]; then - echo chown -R "$ownergroup" "$destination" -fi diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 94b06d63..88019c92 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,112 +3,73 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using rsync +cdist-type__rsync - Mirror directories using ``rsync`` DESCRIPTION ----------- -WARNING: This type is of BETA quality: - -- it has not been tested widely -- interfaces *may* change -- if there is a better approach to solve the problem -> the type may even vanish - -If you are fine with these constraints, please read on. - - -This cdist type allows you to mirror local directories to the -target host using rsync. Rsync will be installed in the manifest of the type. -If group or owner are giveng, a recursive chown will be executed on the -target host. - -A slash will be appended to the source directory so that only the contents -of the directory are taken and not the directory name itself. +The purpose of this type is to bring power of ``rsync`` into ``cdist``. REQUIRED PARAMETERS ------------------- source - Where to take files from + Source directory in local machine. + If source is directory, slash (``/``) will be added to source and destination paths. OPTIONAL PARAMETERS ------------------- -group - Group to chgrp to. +destination + Destination directory. Defaults to ``$__object_id``. owner - User to chown to. + Will be passed to ``rsync`` as ``--chown=OWNER``. + Read ``rsync(1)`` for more details. -destination - Use this as the base destination instead of the object id +group + Will be passed to ``rsync`` as ``--chown=:GROUP``. + Read ``rsync(1)`` for more details. + +mode + Will be passed to ``rsync`` as ``--chmod=MODE``. + Read ``rsync(1)`` for more details. + +options + Defaults to ``--recursive --links --perms --times``. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. remote-user - Use this user instead of the default "root" for rsync operations. + Defaults to ``root``. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -rsync-opts - Use this option to give rsync options with. - See rsync(1) for available options. - Only "--" options are supported. - Write the options without the beginning "--" - Can be specified multiple times. - - -MESSAGES --------- -NONE +option + Pass additional options to ``rsync``. + See ``rsync(1)`` for all possible options. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. EXAMPLES -------- - .. code-block:: sh - # You can use any source directory - __rsync /tmp/testdir \ - --source /etc - - # Use source from type - __rsync /etc \ - --source "$__type/files/package" - - # Allow multiple __rsync objects to write to the same dir - __rsync mystuff \ - --destination /usr/local/bin \ - --source "$__type/files/package" - - __rsync otherstuff \ - --destination /usr/local/bin \ - --source "$__type/files/package2" - - # Use rsync option --exclude - __rsync /tmp/testdir \ - --source /etc \ - --rsync-opts exclude=sshd_conf - - # Use rsync with multiple options --exclude --dry-run - __rsync /tmp/testing \ - --source /home/tester \ - --rsync-opts exclude=id_rsa \ - --rsync-opts dry-run - - -SEE ALSO --------- -:strong:`rsync`\ (1) + __rsync /var/www/example.com \ + --owner root \ + --group www-data \ + --mode 'D750,F640' \ + --source "$__files/example.com/www" AUTHORS ------- -Nico Schottelius +Ander Punnar COPYING ------- -Copyright \(C) 2015 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 9bd44c6d..64fa804e 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,21 +1,3 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options new file mode 100644 index 00000000..d967b110 --- /dev/null +++ b/cdist/conf/type/__rsync/parameter/default/options @@ -0,0 +1 @@ +--recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index ac2b2390..833e9bbe 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,4 +1,6 @@ destination -owner group +mode +options +owner remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index fdb7cd88..01925a15 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -rsync-opts +option From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 18 Jul 2021 16:25:00 +0300 Subject: [PATCH 15/47] use $__remote_exec for RSYNC_RSH --- cdist/conf/type/__rsync/gencode-local | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 612d237e..e9f3c131 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -90,6 +90,8 @@ fi # 4. to understand how that cryptic regex works, please # open rsync manpage and read about --itemize-changes. +export RSYNC_RSH="$__remote_exec" + # shellcheck disable=SC2086 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 @@ -97,4 +99,6 @@ then exit 0 fi +echo "export RSYNC_RSH='$__remote_exec'" + echo "rsync $options $src $remote_user@$__target_host:$dst" From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:41:29 +0200 Subject: [PATCH 16/47] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 284293c1..f0746218 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __rsync: Rewrite (Ander Punnar) + 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:45:19 +0200 Subject: [PATCH 17/47] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f0746218..ea51ac1b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) + * New type: __apt_pin (Daniel Fancsali) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Jul 2021 12:13:23 +0200 Subject: [PATCH 18/47] [explorer/os_version] Convert Devuan ceres to version number Conversion of Devuan ceres to version numbers is done based on Devuan codenames. The version number is the version number of the final release - 0.01. Analogous to Debian. --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..6c94915c 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,12 +18,11 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# # All os variables are lower case # -# -case "$("$__explorer/os")" in +case $("${__explorer:?}/os") +in amazon) cat /etc/system-release ;; @@ -59,7 +59,23 @@ case "$("$__explorer/os")" in esac ;; devuan) - cat /etc/devuan_version + devuan_version=$(cat /etc/devuan_version) + case ${devuan_version} + in + (*/ceres) + # ceres versions don't have a number, so we decode by codename: + case ${devuan_version} + in + (chimaera/ceres) echo 3.99 ;; + (beowulf/ceres) echo 2.99 ;; + (ascii/ceres) echo 1.99 ;; + (*) exit 1 + esac + ;; + (*) + echo "${devuan_version}" + ;; + esac ;; fedora) cat /etc/fedora-release From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Jul 2021 06:38:46 +0200 Subject: [PATCH 19/47] ++changelog --- docs/changelog | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/changelog b/docs/changelog index ea51ac1b..55d7cb73 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) + * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) @@ -146,7 +147,7 @@ next: * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -211,7 +212,7 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1070,9 +1071,9 @@ next: * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 20 Jul 2021 09:03:16 +0200 Subject: [PATCH 20/47] [docs] Bump copyright year to 2021 --- docs/src/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 47765413..a3dfafca 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2020' +copyright = 'ungleich GmbH 2021' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 22 Jul 2021 11:17:41 +0200 Subject: [PATCH 21/47] [cdist.log] Define custom log functions on logging.Logger Define out custom logger functions on logging.Logger so that they are passed on to all other loggers. Also, the logger functions need to take a self argument so that they can log on the corrent Logger. --- cdist/log.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index 113f3b4c..62e457fe 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,25 +36,27 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') + logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(msg, *args, **kwargs): - logging.log(logging.VERBOSE, msg, *args, **kwargs) +def _verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, args, **kwargs) -logging.verbose = _verbose +logging.Logger.verbose = _verbose + logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(msg, *args, **kwargs): - logging.log(logging.TRACE, msg, *args, **kwargs) +def _trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) -logging.trace = _trace +logging.Logger.trace = _trace class CdistFormatter(logging.Formatter): From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 23 Jul 2021 08:06:45 +0200 Subject: [PATCH 22/47] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 55d7cb73..1273e432 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) + * Core: Fix logging bug (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 24 Jul 2021 02:37:58 +0200 Subject: [PATCH 23/47] Improve Makefile compatibility and build docs We now use `$(MAKE)` for subsequent calls to `make`. This means that systems that do not default to GNU make can run `gmake man` and produce the man pages. While there also document a dependency on the rtd theme for sphinx. --- Makefile | 6 +++--- docs/src/cdist-install.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3712511c..89286310 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=make -C $(DOCS_SRC_DIR) man -SPHINXH=make -C $(DOCS_SRC_DIR) html -SPHINXC=make -C $(DOCS_SRC_DIR) clean +SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man +SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html +SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 18863145..390ab9ec 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx (for building html docs and/or the man pages) + * sphinx with the rtd theme (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 24 Jul 2021 12:53:39 +0200 Subject: [PATCH 24/47] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 1273e432..181c4139 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) + * Build: Improve Makefile compatibility (Evilham) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Wed, 28 Jul 2021 12:56:39 +0200 Subject: [PATCH 25/47] [filesystem] Add ubuntu as supported distribution. --- cdist/conf/type/__filesystem/explorer/lsblk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index 9be3c575..d376c09f 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|redhat|suse|gentoo) + alpine|centos|fedora|gentoo|redhat|suse|ubuntu) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 30 Jul 2021 10:30:33 +0200 Subject: [PATCH 26/47] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 181c4139..9fc10b20 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) + * Type __filesystem: Support ubuntu (Joachim Desroches) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 19:50:10 +0200 Subject: [PATCH 27/47] [explorer/os_version] Fix for FreeBSD < 10.0 (again) --- cdist/conf/explorer/os_version | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..cc976608 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -89,7 +89,14 @@ in freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - freebsd-version + if command -v freebsd-version >/dev/null 2>&1 + then + # get userland version + freebsd-version -u + else + # fallback to kernel release for FreeBSD < 10.0 + uname -r + fi ;; *bsd|solaris) uname -r From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 13:20:43 +0200 Subject: [PATCH 28/47] [explorer/os_version] Ubuntu: fall back to os-release/lsb-release files --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..e70fe7f4 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -21,6 +21,17 @@ # All os variables are lower case # +rc_getvar() { + awk -F= -v varname="$2" ' + function unquote(s) { + if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) + return substr(s, 2, length(s) - 2) + else + return s + } + $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" +} + case $("${__explorer:?}/os") in amazon) @@ -114,7 +125,20 @@ in fi ;; ubuntu) - lsb_release -sr + if command -v lsb_release >/dev/null 2>&1 + then + lsb_release -sr + elif test -r /usr/lib/os-release + then + # fallback to /usr/lib/os-release if lsb_release is not present (like + # on minimized Ubuntu installations) + rc_getvar /usr/lib/os-release VERSION_ID + elif test -r /etc/lsb-release + then + # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old + # versions without /usr/lib/os-release. + rc_getvar /etc/lsb-release DISTRIB_RELEASE + fi ;; alpine) cat /etc/alpine-release From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 19:26:55 +0200 Subject: [PATCH 29/47] [explorer/memory] Fix conversion of large numbers (>= 2GiB) At least mawk uses scientific notation when using print for numbers >=2^31 (INT_MAX of a signed 32-bit int). `printf "%.f\n"` works around this. --- cdist/conf/explorer/memory | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 63aba9c6..c6d113cf 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,19 +27,18 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { print $1 * 1000 } - $2 == "MB" { print $1 * 1000 * 1000 } - $2 == "GB" { print $1 * 1000 * 1000 * 1000 } - $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } - $2 == "kiB" { print $1 * 1024 } - $2 == "MiB" { print $1 * 1024 * 1024 } - $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } - $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' + $2 == "kB" { printf "%.f\n", ($1 * 1000) } + $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } + $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } + $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } + $2 == "kiB" { printf "%.f\n", ($1 * 1024) } + $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } + $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } + $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' } bytes2kib() { - set -- "$(cat)" - test "$1" -gt 0 && echo $(($1 / 1024)) + awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' } From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:23:50 +0200 Subject: [PATCH 30/47] [type/__update_alternatives] Secure cdist-defined environment variables with :? --- .../__update_alternatives/explorer/alternatives | 2 +- cdist/conf/type/__update_alternatives/explorer/link | 6 +++--- .../type/__update_alternatives/explorer/path_is | 4 ++-- .../explorer/path_should_state | 2 +- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++------- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index 34aaca56..ecc62f4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "$__object_id" 2>/dev/null \ +update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index 6519e7c2..c6fd1c98 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/$__object_id" ] + if [ ! -f "$altdir/${__object_id:?}" ] then continue fi - link="$( awk 'NR==2' "$altdir/$__object_id" )" + link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" if [ -n "$link" ] then @@ -33,7 +33,7 @@ done if [ -z "$link" ] then - echo "unable to get link for $__object_id" >&2 + echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index fc304d5d..a24bd40e 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,11 +1,11 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ +path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" if [ -z "$path_is" ] then - echo "unable to get current path for $__object_id" >&2 + echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index 59e015c5..b74a7ee8 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "$__object/parameter/path" )" ] +if [ -f "$( cat "${__object:?}/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e393cdef..13666805 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,26 +18,25 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "$__object/explorer/path_is" )" +path_is="$( cat "${__object:?}/explorer/path_is" )" -path_should="$( cat "$__object/parameter/path" )" +path_should="$( cat "${__object:?}/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] +if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ + && [ -z "${__cdist_dry_run+dry run}" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name="$__object_id" +name=${__object_id:?} -alternatives="$( cat "$__object/explorer/alternatives" )" - -if ! echo "$alternatives" | grep -Fxq "$path_should" +if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then if [ ! -f "$__object/parameter/install" ] then From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:25:08 +0200 Subject: [PATCH 31/47] [type/__update_alternatives] dry-run fixes --- cdist/conf/type/__update_alternatives/explorer/link | 5 ++++- .../type/__update_alternatives/explorer/path_is | 5 ++++- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index c6fd1c98..d1087c75 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -31,8 +31,11 @@ do fi done -if [ -z "$link" ] +if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the link + # might be managed by another cdist object (which wasn't executed, + # because dry run…). echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index a24bd40e..9208df7b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -3,8 +3,11 @@ path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] +if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the + # alternative might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index 13666805..e91ea78f 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -38,16 +38,19 @@ name=${__object_id:?} if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then - if [ ! -f "$__object/parameter/install" ] + if [ -f "${__object:?}/parameter/install" ] then + link="$( cat "${__object:?}/explorer/link" )" + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" + elif [ -z "${__cdist_dry_run+dry run}" ] + then + # NOTE: ignore error for dry-runs because a package providing the link + # to be installed might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi - - link="$( cat "$__object/explorer/link" )" - - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:44:04 +0200 Subject: [PATCH 32/47] [type/__update_alternatives] Fix for non-English locales Since update-alternatives(1) is localized, screen scraping its output breaks if the locale is set to non-English. --- cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++-- cdist/conf/type/__update_alternatives/explorer/path_is | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index ecc62f4b..bb1619a9 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ +| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 9208df7b..5cf4fa4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,7 +1,8 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk '/link currently points to/ {print $5}' )" +path_is=$( + LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ + | awk '/link currently points to/ { print $5 }') if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:54:17 +0200 Subject: [PATCH 33/47] [explorer/os_version] Fix for legacy Mac OS X versions --- cdist/conf/explorer/os_version | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..96eca1ee 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -68,7 +68,8 @@ case "$("$__explorer/os")" in cat /etc/gentoo-release ;; macosx) - sw_vers -productVersion + # NOTE: Legacy versions (< 10.3) do not support options + sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 5 Aug 2021 10:27:51 +0200 Subject: [PATCH 34/47] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9fc10b20..f9409d7e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,10 @@ next: * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) * Type __filesystem: Support ubuntu (Joachim Desroches) + * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) + * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) + * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) + * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 21 Jul 2021 13:22:34 +0200 Subject: [PATCH 35/47] [explorer/machine_type] Reimplement --- cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++--- 1 file changed, 904 insertions(+), 82 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1c84f4d7..29f98849 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,8 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Daniel Heule (hda at sfs.biz) -# 2014 Thomas Oettli (otho at sfs.biz) -# 2020 Evilham (contact at evilham.com) +# 2021 Dennis Camera (cdist at dtnr.ch) # # This file is part of cdist. # @@ -19,91 +17,915 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to determine what type of machine the target to be +# configured is (container, virtual machine, bare-metal). +# +# It will print one line for each layer it can detect. +# The format of all lines is: TYPE[ VERB VENDOR] +# +# VERB does not have a special meaning, it is just for better readability. +# +# e.g. +# container on lxc +# virtual by kvm-spapr +# +# The third word of each line can be composed of different parts concatenated with a `-' +# (minus) character, with each component being a specification of the previous, +# e.g.: +# - lxc-libvirt (LXC container, managed by libvirt) +# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) +# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) +# +# If this explorer cannot determine any information it will print nothing. +# -os=$("$__explorer/os") +# Add /sbin and /usr/sbin to the path so we can find system +# binaries like dmidecode. +PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' +PATH="/sbin:/usr/sbin:${PATH}" +export PATH -vendor_string_to_machine_type() { - for vendor in vmware bochs kvm qemu virtualbox bhyve; do - if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then - vendor="kvm" - fi - echo "virtual_by_${vendor}" - exit - fi - done +arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') +uname_s=$(uname -s) + + +is_command() { command -v "$1" >/dev/null 2>&1; } + +is_oneof() ( + x=$1; shift + for y + do + test "${x}" = "${y}" || continue + return 0 + done + return 1 +) + +tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } + +# shellcheck disable=SC2086 +glob_exists() { set -- $1; test -e "$1"; } + +get_dmi_field() { + if is_oneof "${uname_s}" NetBSD + then + case $1 + in + (system-manufacturer) _mib=machdep.dmi.system-vendor ;; + (system-product-name) _mib=machdep.dmi.system-product ;; + (system-version|system-uuid) _mib=machdep.dmi.$1 ;; + (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; + (biod-release-date) _mib=machdep.dmi.bios-date ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + if is_command dmidecode + then + dmidecode -s "$1" + elif test -d "${dmi_sysfs-}" + then + case $1 + in + (system-manufacturer) _filename=sys_vendor ;; + (system-product-name) _filename=product_name ;; + (*) _filename=$(echo "$1" | tr - _) ;; + esac + if test -r "${dmi_sysfs-}/${_filename}" + then + cat "${dmi_sysfs}/${_filename}" + fi + unset _filename + elif test "${uname_s}" = OpenBSD + then + # NOTE: something similar to system-manufacutrer and system-product-name + # is available on OpenBSD in sysctl + case $1 + in + (system-manufacturer) _mib=hw.vendor ;; + (system-product-name) _mib=hw.product ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + return 1 } -case "$os" in - "freebsd") - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" - if [ -n "${vm_guest}" ]; then - if [ "${vm_guest}" = "none" ]; then - echo "physical" - exit - fi - echo "virtual_by_${vm_guest}" - exit - fi - ;; +has_cpuinfo() { test -e /proc/cpuinfo; } - "openbsd") - # OpenBSD can also use the sysctl's: hw.vendor or hw.product. - # Note we can be reasonably sure about a machine being virtualised - # as long as we can identify the virtualisation technology. - # But not so much about it being physical... - # Patches are welcome / reach out if you have better ideas. - for sysctl in hw.vendor hw.product; do - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" - done - ;; +get_sysctl() { + is_command sysctl && sysctl -n "$1" 2>/dev/null +} - *) - # Defaulting to linux for compatibility with previous cdist behaviour - if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit - fi +# Check for container - if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit - fi +has_ct_pid_1() { + test -r /run/systemd/container -o -r /proc/1/environ +} - if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm, bhyve - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - for vendor_file in /sys/class/dmi/id/product_name \ - /sys/class/dmi/id/sys_vendor \ - /sys/class/dmi/id/chasis_vendor; do - if [ -r ${vendor_file} ]; then - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(cat "${vendor_file}")" - fi - done - fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi - fi - ;; -esac +translate_container_name() { + case $1 + in + ('lxc') + echo lxc ;; + ('lxc-libvirt') + echo lxc-libvirt ;; + ('podman') + echo podman ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + return 1 ;; + esac + return 0 +} -echo "unknown" +check_ct_pid_1() { + if test -r /run/systemd/container + then + translate_container_name "$(head -n1 /run/systemd/container)" \ + && return 0 + fi + + if test -r /proc/1/environ + then + translate_container_name "$( + LC_ALL=C tr '\000' '\n' /dev/null + then + # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + echo wsl + elif test -d /var/.cagefs + then + # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs + # CageFS is not "really" a container, but it isn't a chroot either. + echo cagefs + elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status + then + # Linux-VServer + if grep -q -x -F 'VxID: 0' /proc/self/status + then + # host + return 1 + else + # guest + echo linux_vserver + fi + else + return 1 + fi +} + +check_ct_os_specific() ( + if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 + then + # FreeBSD jail + echo jail + return 0 + fi + + if is_command zonename && test "$(zonename)" != global + then + # Solaris zone + echo zone + return 0 + fi + + return 1 +) + + +# Check for hypervisor + +guess_hypervisor_from_cpu_model() { + case $1 + in + (*\ KVM\ *) + echo kvm ;; + (*\ QEMU\ *|QEMU\ *) + echo qemu ;; + (*) + return 1 ;; + esac +} + +has_vm_cpuinfo() { has_cpuinfo; } + +check_vm_cpuinfo() { + if grep -q -F 'User Mode Linux' /proc/cpuinfo \ + || grep -q -F 'UML' /proc/cpuinfo + then + # User Mode Linux + echo uml + elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo + then + # IBM PowerVM Lx86 (Linux/x86 emulator) + echo powervm_lx86 + elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo + then + # IBM SystemZ (S/390) + if test -f /proc/sysinfo + then + if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo + then + echo kvm-s390 + return 0 + elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo + then + echo zvm + return 0 + elif grep -q -e '^LPAR ' /proc/sysinfo + then + echo zvm-lpar + return 0 + fi + fi + return 1 + else + if grep -q -e '^model name.*:' /proc/cpuinfo + then + sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ + | while read -r _cpu_model + do + guess_hypervisor_from_cpu_model "${_cpu_model}" + done \ + | sort \ + | uniq -c \ + | awk ' + { if ($1 > most_c) { most_c = $1; most_s = $2 } } + END { + if (most_s) print most_s + exit !most_s + }' \ + && return 0 + fi + return 1 + fi +} + +check_vm_arch_specific() { + case ${arch} + in + (ppc64|ppc64le) + # Check PPC64 LPAR, KVM + + # example /proc/cpuinfo line indicating 'not baremetal' + # platform : pSeries + # + # example /proc/ppc64/lparcfg systemtype line + # system_type=IBM pSeries (emulated by qemu) + + if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo + then + if test -e /proc/ppc64/lparcfg + then + # Assume LPAR, now detect shared or dedicated + if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg + then + echo powervm-shared + return 0 + else + echo powervm-dedicated + return 0 + fi + fi + fi + ;; + (sparc*) + # Check for SPARC LDoms + + if test -e /dev/mdesc + then + if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp + then + # control LDom + return 1 + else + # guest LDom + echo ldom-sparc + fi + + # MDPROP=/usr/lib/ldoms/mdprop.py + # if test -x "${MDPROP}" + # then + # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" + # then + # echo ldoms-root + # echo ldoms-io + # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" + # then + # echo ldoms-io + # fi + # fi + return 0 + fi + ;; + (i?86|x86*|amd64|i86pc) + # Check CPUID + # + # Many fullvirt hypervisors give an indication through CPUID. Use + # the virt-what helper program to get this information if available. + + for CPUID_HELPER in \ + $(command -v virt-what-cpuid-helper 2>/dev/null) \ + /usr/lib/x86_64-*/virt-what-cpuid-helper \ + /usr/lib/i?86-*/virt-what-cpuid-helper \ + /usr/lib/virt-what/virt-what-cpuid-helper + do + if test -x "${CPUID_HELPER:?}"; then break; fi + done + + if test -x "${CPUID_HELPER-}" + then + case $(command "${CPUID_HELPER}") + in + ('bhyve bhyve ') + echo bhyve + ;; + ('LKVMLKVMLKVM') + echo lkvm + ;; + ('KVMKVMKVM') + echo kvm + ;; + ('TCGTCGTCGTCG') + echo qemu-tcg + ;; + ('Microsoft Hv') + # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx + echo hyperv + ;; + ('OpenBSDVMM58') + # OpenBSD/VMM + echo openbsd_vmm + ;; + ('VMwareVMware') + # check added by Chetan Loke. + echo vmware + ;; + ('XenVMMXenVMM') + if has dmi + then + # https://access.redhat.com/solutions/222903 + echo xen-hvm + else + echo xen-paravirt + fi + ;; + (*) + return 1 ;; + esac + return 0 + fi + + unset CPUID_HELPER + + # VMM CPUID flag denotes that this system is running under a VMM + if is_oneof "${uname_s}" Darwin + then + get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ + && return 0 + fi + if has_cpuinfo \ + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + then + return 0 + fi + ;; + (ia64) + if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend + then + # PV-on-HVM drivers installed in a Xen guest + echo xen-hvm + return 0 + fi + ;; + esac + return 1 +} + +has_vm_dmi() { + # Check for various products in SMBIOS/DMI. + # Note that DMI doesn't exist on all architectures (only x86 and some ARM). + # On other architectures the $dmi variable will be empty. + + if test -d /sys/class/dmi/id/ + then + dmi_sysfs=/sys/class/dmi/id + elif test -d /sys/devices/virtual/dmi/id/ + then + dmi_sysfs=/sys/devices/virtual/dmi/id + fi + + # shellcheck disable=SC2015 + { + is_command dmidecode \ + && ( + # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line + dmi_out=$(dmidecode 2>&1) \ + && ! printf '%s\n' "${dmi_out}" \ + | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' + ) \ + || test -d "${dmi_sysfs}" + } +} + +check_vm_dmi() { + case $(get_dmi_field system-product-name) + in + (*.metal) + if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' + then + # AWS EC2 bare metal -> no virtualisation + return 1 + fi + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('Google Compute Engine') + echo gce + return 0 + ;; + ('RHEV Hypervisor') + # Red Hat Enterprise Virtualization + echo rhev + return 0 + ;; + ('KVM'|'Bochs'|'KVM Virtual Machine') + echo kvm + return 0 + ;; + ('Parallels Virtual Platform') + echo parallels + return 0 + ;; + ('VirtualBox') + echo virtualbox + return 0 + ;; + ('VMware Virtual Platform') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field system-manufacturer) + in + ('Alibaba'*) + case $(get_dmi_field system-product-name) + in + ('Alibaba Cloud ECS') + echo alibaba-ecs + ;; + (*) + echo alibaba + ;; + esac + return 0 + ;; + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('innotek GmbH'|'Oracle Corporation') + echo virtualbox + return 0 + ;; + ('Joyent') + if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' + then + # SmartOS KVM + echo kvm-smartdc_hvm + return 0 + fi + ;; + ('Microsoft Corporation'*) + if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' + then + if test -e /proc/irq/7/hyperv \ + || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null + then + echo hyperv + return 0 + fi + + case $(get_dmi_field system-version) + in + (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) + echo virtualpc + return 0 + ;; + (*) + echo hyperv + return 0 + ;; + esac + fi + ;; + ('Nutanix') + # Nutanix AHV. Similar to KVM. + if test "$(get_dmi_field system-product-name)" = 'AHV' + then + echo nutanix_ahv + return 0 + fi + ;; + ('oVirt') + echo ovirt + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('QEMU') + echo qemu + return 0 + ;; + ('VMware, Inc.') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field bios-vendor) + in + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('innotek GmbH') + echo virtualbox + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('Xen') + if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' + then + # AWS on Xen + echo aws-xen + return 0 + fi + ;; + esac + + return 1 +} + +check_vm_hyp_specific() { + if is_command vmware-checkvm && vmware-checkvm >/dev/null + then + # vmware-checkvm is provided by VMware's open-vm-tools + echo vmware + return 0 + elif test -d /proc/xen + then + test -r /proc/xen/capabilities && + if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null + then + # Xen dom0 + return 1 + else + # Xen domU + echo xen + return 0 + fi + fi + return 1 +} + +has_vm_dt() { + # OpenFirmware/Das U-Boot device-tree + test -d /proc/device-tree +} + +check_vm_dt() { + case ${arch} + in + (arm|aarch64) + if test -r /proc/device-tree/hypervisor/compatible + then + if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible + then + echo xen + return 0 + elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible + then + # e.g. VMware ESXi on ARM + echo vmware + return 0 + fi + fi + if glob_exists /proc/device-tree/fw-cfg@*/compatible + then + # qemu,fw-cfg-mmio + sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 + return 0 + fi + if grep -q -F 'dummy-virt' /proc/device-tree/compatible + then + echo lkvm + return 0 + fi + ;; + (ppc64*) + if test -d /proc/device-tree/hypervisor \ + && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible + then + # We are running as a spapr KVM guest on ppc64 + echo kvm-spapr + return 0 + fi + if test -r /proc/device-tree/ibm,partition-name \ + && test -r /proc/device-tree/hmc-managed\? \ + && test -r /proc/device-tree/chosen/qemu,graphic-width + then + echo powervm + fi + ;; + esac + return 1 +} + +has_vm_sys_hypervisor() { + test -d /sys/hypervisor/ +} + +check_vm_sys_hypervisor() { + test -r /sys/hypervisor/type && + case $(head -n1 /sys/hypervisor/type) + in + (xen) + # Ordinary kernel with pv_ops. There does not seem to be + # enough information at present to tell whether this is dom0 + # or domU. + echo xen + return 0 + ;; + esac + return 1 +} + +check_vm_os_specific() { + _hyp_generic=false + + case ${uname_s} + in + (Darwin) + if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ + && test "${hv_vmm_present}" -ne 0 + then + _hyp_generic=true + fi + ;; + (FreeBSD) + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest=$(get_sysctl kern.vm_guest | tolower) && + case ${vm_guest} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${vm_guest} + in + (hv) echo hyperv ;; + (vbox) echo virtualbox ;; + (*) echo "${vm_guest}" ;; + esac + return 0 + ;; + esac + ;; + (NetBSD) + machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && + case ${machdep_hv} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${machdep_hv} + in + (hyper-v) echo hyperv ;; + (xenhvm*) echo xen-hvm ;; + (xenpv*) echo xen-pv ;; + (xen*) echo xen ;; + (*) echo "${machdep_hv}" ;; + esac + return 0 + ;; + esac + ;; + (OpenBSD) + if is_command hostctl && glob_exists /dev/pvbus[0-9]* + then + for _pvbus in /dev/pvbus[0-9]* + do + _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue + case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') + in + (KVM) echo kvm ;; + (Hyper-V) echo hyperv ;; + (VMware) echo vmware ;; + (Xen) echo xen ;; + (bhyve) echo bhyve ;; + (OpenBSD) echo openbsd_vmm ;; + esac + return 0 + done + fi + ;; + (SunOS) + diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') + # NOTE: Don't use -e or -F in Solaris grep + if printf '%s\n' "${diag_conf}" | grep -q -i QEMU + then + echo qemu + return 0 + elif printf '%s\n' "${diag_conf}" | grep -q -i VMware + then + echo vmware + return 0 + fi + ;; + (Linux) + if is_command dmesg + then + while read -r line + do + case ${line} + in + ('Booting paravirtualized kernel on ') + case $(expr "${line}" : '.* kernel on \(.*\)') + in + ('Xen') + echo xen-pv; return 0 ;; + ('bare hardware') + return 1 ;; + esac + ;; + ('Hypervisor detected') + case $(expr "${line}" : '.*: *\(.*\)') + in + ('ACRN') + echo acrn ;; + ('Jailhouse') + echo jailhouse ;; + ('KVM') + echo kvm ;; + ('Microsoft Hyper-V') + echo hyperv ;; + ('VMware') + echo vmware ;; + ('Xen HVM') + echo xen-hvm ;; + ('Xen PV') + echo xen-pv ;; + esac + return 0 + ;; + (lpar:*' under hypervisor') + return 0 ;; + esac + done <<-EOF + $(dmesg 2>/dev/null | awk ' + /Booting paravirtualized kernel on / + /Hypervisor detected: / + /lpar: .* under hypervisor/ + ') + EOF + fi + esac + + # Try to guess hypervisor based on CPU model (sysctl hw.model if available) + if cpu_model=$(get_sysctl hw.model) + then + guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 + fi + + if ${_hyp_generic} + then + # cannot say which hypervisor, but one was detected + return 0 + else + return 1 + fi +} + +run_stage() { + if type "has_$1_$2" >/dev/null 2>&1 + then + "has_$1_$2" + else + true + fi \ + && "check_$1_$2" +} + + +# Execute container stages + +for stage in \ + pid_1 cgroup files os_specific +do + ctengine=$(run_stage ct ${stage}) || continue + is_contained=true + if test -n "${ctengine}" + then + echo container on "${ctengine}" + break + fi +done +if ${is_contained:-false} && test -z "${ctengine}" +then + # none of the stages could determine the specific container engine, but + # we are running in some container. + echo container +fi + + +# Execute virtual machine / hypervisor stages + +for stage in \ + os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific +do + hypervisor=$(run_stage vm ${stage}) || continue + is_virtual=true + if test -n "${hypervisor}" + then + echo virtual by "${hypervisor}" + break + fi +done +if ${is_virtual:-false} && test -z "${hypervisor}" +then + # none of the stages could determine the specific hypervisor, but + # we are virtual. + echo virtual +fi From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:29:41 +0200 Subject: [PATCH 36/47] [explorer/machine_type] Print top most machine layer as first line (fallback to physical) --- cdist/conf/explorer/machine_type | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 29f98849..90c441da 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -26,17 +26,19 @@ # VERB does not have a special meaning, it is just for better readability. # # e.g. +# container # container on lxc # virtual by kvm-spapr # -# The third word of each line can be composed of different parts concatenated with a `-' -# (minus) character, with each component being a specification of the previous, -# e.g.: +# The third word of each line (except the first) can be composed of different +# parts concatenated with a `-' (minus) character, with each component being +# a specification of the previous, e.g.: # - lxc-libvirt (LXC container, managed by libvirt) # - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) # - xen-hvm / xen-pv (Xen HVM vs para-virtualization) # -# If this explorer cannot determine any information it will print nothing. +# If this explorer cannot collect enough information about virtualization it +# will fall back to 'physical'. # # Add /sbin and /usr/sbin to the path so we can find system @@ -121,6 +123,10 @@ get_sysctl() { is_command sysctl && sysctl -n "$1" 2>/dev/null } +detected_layer() { + test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" +} + # Check for container @@ -895,6 +901,7 @@ for stage in \ pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue + detected_layer 'container' is_contained=true if test -n "${ctengine}" then @@ -916,6 +923,7 @@ for stage in \ os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue + detected_layer 'virtual machine' is_virtual=true if test -n "${hypervisor}" then @@ -929,3 +937,8 @@ then # we are virtual. echo virtual fi + + +# Fallback + +detected_layer physical From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:57:24 +0200 Subject: [PATCH 37/47] [explorer/machine_type] Remove CPUID check it's a lot of code and depends on a binary helper unlikely to be installed. --- cdist/conf/explorer/machine_type | 59 -------------------------------- 1 file changed, 59 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 90c441da..7ce035e3 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -379,65 +379,6 @@ check_vm_arch_specific() { fi ;; (i?86|x86*|amd64|i86pc) - # Check CPUID - # - # Many fullvirt hypervisors give an indication through CPUID. Use - # the virt-what helper program to get this information if available. - - for CPUID_HELPER in \ - $(command -v virt-what-cpuid-helper 2>/dev/null) \ - /usr/lib/x86_64-*/virt-what-cpuid-helper \ - /usr/lib/i?86-*/virt-what-cpuid-helper \ - /usr/lib/virt-what/virt-what-cpuid-helper - do - if test -x "${CPUID_HELPER:?}"; then break; fi - done - - if test -x "${CPUID_HELPER-}" - then - case $(command "${CPUID_HELPER}") - in - ('bhyve bhyve ') - echo bhyve - ;; - ('LKVMLKVMLKVM') - echo lkvm - ;; - ('KVMKVMKVM') - echo kvm - ;; - ('TCGTCGTCGTCG') - echo qemu-tcg - ;; - ('Microsoft Hv') - # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx - echo hyperv - ;; - ('OpenBSDVMM58') - # OpenBSD/VMM - echo openbsd_vmm - ;; - ('VMwareVMware') - # check added by Chetan Loke. - echo vmware - ;; - ('XenVMMXenVMM') - if has dmi - then - # https://access.redhat.com/solutions/222903 - echo xen-hvm - else - echo xen-paravirt - fi - ;; - (*) - return 1 ;; - esac - return 0 - fi - - unset CPUID_HELPER - # VMM CPUID flag denotes that this system is running under a VMM if is_oneof "${uname_s}" Darwin then From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 21:29:24 +0200 Subject: [PATCH 38/47] [explorer/machine_type] Use systemd-detect-virt (if available) to detect containers and VMs --- cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 7ce035e3..1a38fda0 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -130,6 +130,25 @@ detected_layer() { # Check for container +has_ct_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -c' +} + +check_ct_systemd() ( + _ctengine=$(systemd-detect-virt -c 2>/dev/null) && + case ${_ctengine} + in + (''|'none') + return 1 ;; + ('container-other') + return 0 ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + echo "${_ctengine}" ;; + esac +) + has_ct_pid_1() { test -r /run/systemd/container -o -r /proc/1/environ } @@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() { esac } +has_vm_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' +} + +check_vm_systemd() ( + _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && + case ${_hypervisor} + in + (''|'none') + return 1 ;; + ('amazon') + echo aws ;; + ('bochs') + echo kvm ;; + ('microsoft') + # assumption + echo hyperv ;; + ('oracle') + echo virtualbox ;; + ('vm-other') + return 0 ;; + (*) + echo "${_hypervisor}" ;; + esac +) + has_vm_cpuinfo() { has_cpuinfo; } check_vm_cpuinfo() { @@ -839,7 +884,7 @@ run_stage() { # Execute container stages for stage in \ - pid_1 cgroup files os_specific + systemd pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue detected_layer 'container' @@ -861,7 +906,7 @@ fi # Execute virtual machine / hypervisor stages for stage in \ - os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific + systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue detected_layer 'virtual machine' From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 22:01:28 +0200 Subject: [PATCH 39/47] [explorer/machine_type] Implement chroot detection --- cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1a38fda0..10c914ba 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -53,6 +53,21 @@ uname_s=$(uname -s) is_command() { command -v "$1" >/dev/null 2>&1; } +files_same() { + # shellcheck disable=SC2012 + LC_ALL=C df -P "$1" "$2" 2>/dev/null | { + read -r _ # skip header line + read -r fs1 _ _ _ _ mp1 + read -r fs2 _ _ _ _ mp2 + test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 + } && + ls -1Ldi "$1" "$2" 2>/dev/null | { + read -r ino1 _ + read -r ino2 _ + test "${ino1}" = "${ino2}" || return 1 + } +} + is_oneof() ( x=$1; shift for y @@ -128,6 +143,32 @@ detected_layer() { } +# Check for chroot + +has_chroot_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' +} + +check_chroot_systemd() { + systemd-detect-virt -r +} + +has_chroot_debian_ischroot() { + is_command ischroot +} + +check_chroot_debian_ischroot() { + ischroot --default-false +} + +has_chroot_procfs() { + test -d /proc/ +} + +check_chroot_procfs() { + test -e /proc/1/root && ! files_same /proc/1/root / +} + # Check for container has_ct_systemd() { @@ -881,6 +922,18 @@ run_stage() { } +# Execute chroot stages + +for stage in \ + procfs debian_ischroot systemd +do + run_stage chroot ${stage} || continue + detected_layer 'chroot' + echo chroot + break +done + + # Execute container stages for stage in \ From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 16:40:20 +0200 Subject: [PATCH 40/47] [explorer/machine_type] Try to detect chroot path --- cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 10c914ba..fa68ec4d 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -166,7 +166,28 @@ has_chroot_procfs() { } check_chroot_procfs() { - test -e /proc/1/root && ! files_same /proc/1/root / + if test -e /proc/1/root && ! files_same /proc/1/root / + then + # try to determine where the chroot has been mounted + ( + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # Get mount point + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + fi + ) + return 0 + fi + return 1 } # Check for container @@ -927,11 +948,20 @@ run_stage() { for stage in \ procfs debian_ischroot systemd do - run_stage chroot ${stage} || continue + chrootpnt=$(run_stage chroot ${stage}) || continue + is_chrooted=true detected_layer 'chroot' - echo chroot - break + if test -n "${chrootpnt}" + then + echo chroot at "${chrootpnt}" + break + fi done +if ${is_chrooted:-false} && test -z "${chrootpnt}" +then + # could determine chroot, but not its mount point + echo chroot +fi # Execute container stages From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 23:09:02 +0200 Subject: [PATCH 41/47] [explorer/machine_type] Implement chroot detection using /proc/.../mountinfo --- cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index fa68ec4d..00646c75 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -165,30 +165,52 @@ has_chroot_procfs() { test -d /proc/ } -check_chroot_procfs() { +check_chroot_procfs() ( + is_chroot=false # default if test -e /proc/1/root && ! files_same /proc/1/root / then - # try to determine where the chroot has been mounted - ( - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # Get mount point - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - fi - ) - return 0 + is_chroot=true fi - return 1 -} + if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo + then + has_mountinfo=true + cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true + fi + + if ${is_chroot} + then + # try to determine where the chroot has been mounted + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # escape chroot and find mount point by inode + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + elif ${has_mountinfo} + then + while read -r mntid _ _ _ cmntpnt _ + do + read -r _ _ _ _ hmntpnt _ <<-EOF + $(grep -e "^$((mntid)) " /proc/1/mountinfo) + EOF + printf '%s\n' "${hmntpnt%${cmntpnt}}" + done Date: Mon, 23 Aug 2021 09:57:20 +0300 Subject: [PATCH 42/47] [explorer/os_version] add new debian code names: bookworm and trixie --- cdist/conf/explorer/os_version | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 7bc6dd6b..aea3c43f 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -54,6 +54,8 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in + trixie) echo 12.99 ;; + bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 23 Aug 2021 10:44:48 +0300 Subject: [PATCH 43/47] [explorer/os_version] use 99.99 as fallback for unknown code names in */sid --- cdist/conf/explorer/os_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index aea3c43f..bbc9e4f0 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -63,7 +63,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) exit 1 + *) echo 99.99 ;; esac ;; *) From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 08:09:47 +0200 Subject: [PATCH 44/47] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f9409d7e..8507c663 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) + * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:32:44 +0200 Subject: [PATCH 45/47] Update shellcheck disable --- cdist/conf/type/__apt_backports/manifest | 1 + cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 + cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 3 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index bc47d8de..6fcd9212 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,6 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 3f88f6bc..ca9aa45a 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,6 +37,7 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 + # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index cec25746..9694a64b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,6 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1090 +# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:25:49 +0200 Subject: [PATCH 46/47] Release 6.9.8 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8507c663..dcdc4b3d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:47:50 +0200 Subject: [PATCH 47/47] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index dcdc4b3d..693d028f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Explorer machine_type: Rewrite (Dennis Camera) + 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali)