From 0734288483700e8e10cebd87c797b625aa83d55e Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Sun, 21 Feb 2021 19:59:57 +0000 Subject: [PATCH 001/128] First draft of __apt_pin --- cdist/conf/type/__apt_pin/man.rst | 53 ++++++++++++++++ cdist/conf/type/__apt_pin/manifest | 63 +++++++++++++++++++ .../type/__apt_pin/parameter/default/package | 1 + .../type/__apt_pin/parameter/default/state | 1 + cdist/conf/type/__apt_pin/parameter/optional | 2 + cdist/conf/type/__apt_pin/parameter/required | 2 + 6 files changed, 122 insertions(+) create mode 100644 cdist/conf/type/__apt_pin/man.rst create mode 100755 cdist/conf/type/__apt_pin/manifest create mode 100644 cdist/conf/type/__apt_pin/parameter/default/package create mode 100644 cdist/conf/type/__apt_pin/parameter/default/state create mode 100644 cdist/conf/type/__apt_pin/parameter/optional create mode 100644 cdist/conf/type/__apt_pin/parameter/required diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst new file mode 100644 index 00000000..7fcae6f8 --- /dev/null +++ b/cdist/conf/type/__apt_pin/man.rst @@ -0,0 +1,53 @@ +cdist-type__apt_pin(7) +====================== + +NAME +---- +cdist-type__apt_pin - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __apt_pin + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest new file mode 100755 index 00000000..8dd9770d --- /dev/null +++ b/cdist/conf/type/__apt_pin/manifest @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") +state="$(cat "$__object/parameter/state")" +package="$(cat "$__object/parameter/package")" +distribution="$(cat "$__object/parameter/distribution")" +priority="$(cat "$__object/parameter/priority")" + + +case "$os" in + debian|ubuntu|devuan) + ;; + *) + printf "This type is specific to Debian and it's derivatives" >&2 + printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 + exit 1 + ;; +esac + +if [ "$package" = "*" ]; then + name="default" + +else + name="$__object_id" +fi + +case $distribution in + stabletesting|unsatbel|experimental) + pin="release a=$distribution" + ;; + *) + pin="release n=$distribution" + ;; +esac + + +__file /etc/apt/preferences.d/$name \ + --owner root --group root --mode 0644 \ + --state "$state" \ + --source - << EOF +Package: $package +Pin: $pin +Pin-Priority: $priority +EOF diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package new file mode 100644 index 00000000..72e8ffc0 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/package @@ -0,0 +1 @@ +* diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional new file mode 100644 index 00000000..52f01fd2 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -0,0 +1,2 @@ +state +package diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required new file mode 100644 index 00000000..4b4e9741 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -0,0 +1,2 @@ +distribution +priority From 1a74470c4d9b30e41a72fbfc084dc54ea44e643b Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 09:37:36 +0000 Subject: [PATCH 002/128] __apt_pin: Always use $__object_id as preferences.d filename --- cdist/conf/type/__apt_pin/manifest | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 8dd9770d..162b523f 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -36,12 +36,7 @@ case "$os" in ;; esac -if [ "$package" = "*" ]; then - name="default" - -else - name="$__object_id" -fi +name="$__object_id" case $distribution in stabletesting|unsatbel|experimental) From dc66efa690e15ff32d6836e23baa1cbec5eee1ed Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Tue, 23 Feb 2021 11:59:09 +0000 Subject: [PATCH 003/128] Fix shellcheck issues --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 162b523f..b1372ad0 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -48,7 +48,7 @@ case $distribution in esac -__file /etc/apt/preferences.d/$name \ +__file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF From acf9bf91f17fb4ad0e5813c6e4b0b40fc7e027b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 08:55:14 +0200 Subject: [PATCH 004/128] [scanner] error to stderr and exit when scapy is not available --- cdist/scan/commandline.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index eca4cf13..0d7fb0ca 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,6 +20,7 @@ # import logging +import sys log = logging.getLogger("scan") @@ -31,7 +32,9 @@ def commandline(args): try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed') + print('cdist scan requires scapy to be installed! Exiting.', + file=sys.stderr) + sys.exit(1) processes = [] From a4464209b6741f2c1764aa432c56cfa8128852a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 09:29:53 +0200 Subject: [PATCH 005/128] [scanner] add minimal error handling, consolidate CLI args processing --- cdist/argparse.py | 10 ++++-- cdist/scan/commandline.py | 19 ++++++----- cdist/scan/scan.py | 71 +++++++++++---------------------------- 3 files changed, 37 insertions(+), 63 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index cadac39a..153e7864 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -492,12 +492,16 @@ def get_parsers(): help='Try to configure detected hosts') parser['scan'].add_argument( '-I', '--interfaces', - action='append', default=[], + action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( '-d', '--delay', - action='store', default=3600, - help='How long to wait before reconfiguring after last try') + action='store', default=3600, type=int, + help='How long (seconds) to wait before reconfiguring after last try') + parser['scan'].add_argument( + '-t', '--trigger-delay', + action='store', default=5, type=int, + help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 0d7fb0ca..dead5292 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -24,26 +24,29 @@ import sys log = logging.getLogger("scan") - -# define this outside of the class to not handle scapy import errors by default +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): log.debug(args) + # Check if we have the optional scapy dependency available. try: import cdist.scan.scan as scan except ModuleNotFoundError: - print('cdist scan requires scapy to be installed! Exiting.', - file=sys.stderr) + log.error('cdist scan requires scapy to be installed. Exiting.') sys.exit(1) - processes = [] - + # Default operation mode. if not args.mode: - # By default scan and trigger, but do not call any action + # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + # We run each component in a separate process since they + # must not block on each other. + processes = [] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces) + t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index faee8a56..633a5c06 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -61,20 +61,22 @@ import datetime import cdist.config +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") - class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces=None, verbose=False): + def __init__(self, interfaces, sleeptime, verbose=False): self.interfaces = interfaces + + # Used by scapy / send in trigger/2. self.verbose = verbose - # Wait 5 seconds before triggering again - FIXME: add parameter - self.sleeptime = 5 + # Delay in seconds between sent ICMPv6EchoRequests. + self.sleeptime = sleeptime def start(self): self.processes = [] @@ -93,9 +95,12 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() - log.debug("Sending request on %s", interface) - send(packet, verbose=self.verbose) + try: + log.debug("Sending ICMPv6EchoRequest on %s", interface) + packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + send(packet, verbose=self.verbose) + except Exception as e: + log.error( "Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -103,7 +108,7 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces=None, args=None, outdir=None): + def __init__(self, interfaces, args=None, outdir=None): self.interfaces = interfaces if outdir: @@ -148,47 +153,9 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - - -if __name__ == '__main__': - t = Trigger(interfaces=["wlan0"]) - t.start() - - # Scanner can listen on many interfaces at the same time - s = Scanner(interfaces=["wlan0"]) - s.scan() - - # Join back the trigger processes - t.join() - - # Test in my lan shows: - # [18:48] bridge:cdist% ls -1d fe80::* - # fe80::142d:f0a5:725b:1103 - # fe80::20d:b9ff:fe49:ac11 - # fe80::20d:b9ff:fe4c:547d - # fe80::219:d2ff:feb2:2e12 - # fe80::21b:fcff:feee:f446 - # fe80::21b:fcff:feee:f45c - # fe80::21b:fcff:feee:f4b1 - # fe80::21b:fcff:feee:f4ba - # fe80::21b:fcff:feee:f4bc - # fe80::21b:fcff:feee:f4c1 - # fe80::21d:72ff:fe86:46b - # fe80::42b0:34ff:fe6f:f6f0 - # fe80::42b0:34ff:fe6f:f863 - # fe80::42b0:34ff:fe6f:f9b2 - # fe80::4a5d:60ff:fea1:e55f - # fe80::77a3:5e3f:82cc:f2e5 - # fe80::9e93:4eff:fe6c:c1f4 - # fe80::ba69:f4ff:fec5:6041 - # fe80::ba69:f4ff:fec5:8db7 - # fe80::bad8:12ff:fe65:313d - # fe80::bad8:12ff:fe65:d9b1 - # fe80::ce2d:e0ff:fed4:2611 - # fe80::ce32:e5ff:fe79:7ea7 - # fe80::d66d:6dff:fe33:e00 - # fe80::e2ff:f7ff:fe00:20e6 - # fe80::f29f:c2ff:fe7c:275e + try: + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + except Exception as e: + log.error( "Could not start listener: %s", e) From bb24d632d62da8390dd1a724fc325a57526da40d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 22 Apr 2021 10:20:49 +0200 Subject: [PATCH 006/128] [scanner] implement the --list flag --- cdist/argparse.py | 4 +++ cdist/scan/commandline.py | 63 ++++++++++++++++++++++++++++----------- cdist/scan/scan.py | 17 +++++++++++ 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 153e7864..f390a974 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -486,6 +486,10 @@ def get_parsers(): '-m', '--mode', help='Which modes should run', action='append', default=[], choices=['scan', 'trigger']) + parser['scan'].add_argument( + '--list', + action='store_true', + help='List the known hosts and exit') parser['scan'].add_argument( '--config', action='store_true', diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index dead5292..331694e4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -21,26 +21,11 @@ import logging import sys +from datetime import datetime log = logging.getLogger("scan") -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - +def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] @@ -59,3 +44,47 @@ def commandline(args): for process in processes: process.join() + +def list(scan, args): + s = scan.Scanner(interfaces=args.interfaces, args=args) + hosts = s.list() + + # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + + # 6 colons. + ipv6_max_size = 8 * 4 + 10 + # We format dates as follow: YYYY-MM-DD HH:MM:SS + date_max_size = 8 + 2 + 6 + 2 + + print("{} | {}".format( + 'link-local address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size))) + print('=' * (ipv6_max_size + 3 + date_max_size)) + for addr in hosts: + last_seen = datetime.strftime( + datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), + '%Y-%m-%d %H:%M:%S') + print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). +def commandline(args): + log.debug(args) + + # Check if we have the optional scapy dependency available. + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + log.error('cdist scan requires scapy to be installed. Exiting.') + sys.exit(1) + + # Set default operation mode. + if not args.mode: + # By default scan and trigger, but do not call any action. + args.mode = ['scan', 'trigger', ] + + # Print known hosts and exit is --list is specified - do not start + # the scanner. + if args.list: + list(scan, args) + else: + run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 633a5c06..f3976370 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -132,6 +132,23 @@ class Scanner(object): with open(fname, "w") as fd: fd.write(f"{now}\n") + def list(self): + hosts = dict() + for linklocal_addr in os.listdir(self.outdir): + workdir = os.path.join(self.outdir, linklocal_addr) + # We ignore any (unexpected) file in this directory. + if os.path.isdir(workdir): + last_seen='-' + last_seen_file = os.path.join(workdir, 'last_seen') + if os.path.isfile(last_seen_file): + with open(last_seen_file, "r") as fd: + last_seen = fd.readline() + + hosts[linklocal_addr] = {'last_seen': last_seen} + + return hosts + + def config(self): """ Configure a host From 13e2ad175f01b6cfbdb21ee50e85b6f3ba6fe750 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 25 Apr 2021 12:45:34 +0200 Subject: [PATCH 007/128] [scanner] add host class, name mapper and pre-config logic --- cdist/argparse.py | 6 +- cdist/scan/commandline.py | 37 ++++++++---- cdist/scan/scan.py | 122 +++++++++++++++++++++++++++++--------- 3 files changed, 124 insertions(+), 41 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f390a974..bedb23ac 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,7 +485,7 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger']) + choices=['scan', 'trigger', 'config']) parser['scan'].add_argument( '--list', action='store_true', @@ -498,6 +498,10 @@ def get_parsers(): '-I', '--interfaces', action='append', default=[], required=True, help='On which interfaces to scan/trigger') + parser['scan'].add_argument( + '--name-mapper', + action='store', default=None, + help='Map addresses to names, required for config mode') parser['scan'].add_argument( '-d', '--delay', action='store', default=3600, type=int, diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 331694e4..1a0ab0a7 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -37,7 +37,10 @@ def run(scan, args): log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner( + autoconfigure='config' in args.mode, + interfaces=args.interfaces, + name_mapper=args.name_mapper) s.start() processes.append(s) log.debug("Scanner started") @@ -46,24 +49,27 @@ def run(scan, args): process.join() def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + # 6 colons. ipv6_max_size = 8 * 4 + 10 - # We format dates as follow: YYYY-MM-DD HH:MM:SS - date_max_size = 8 + 2 + 6 + 2 + date_max_size = len(datetime.now().strftime(scan.datetime_format)) + name_max_size = 25 - print("{} | {}".format( - 'link-local address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size))) - print('=' * (ipv6_max_size + 3 + date_max_size)) - for addr in hosts: - last_seen = datetime.strftime( - datetime.strptime(hosts[addr]['last_seen'].strip(), '%Y-%m-%d %H:%M:%S.%f'), - '%Y-%m-%d %H:%M:%S') - print("{} | {}".format(addr.ljust(ipv6_max_size),last_seen.ljust(date_max_size))) + print("{} | {} | {} | {}".format( + 'name'.ljust(name_max_size), + 'address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size), + 'last configured'.ljust(date_max_size))) + print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) + for host in hosts: + print("{} | {} | {} | {}".format( + host.name(default='-').ljust(name_max_size), + host.address().ljust(ipv6_max_size), + host.last_seen().ljust(date_max_size), + host.last_configured().ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). @@ -82,6 +88,11 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] + if 'config' in args.mode and args.name_mapper == None: + print('--name-mapper must be specified for scanner config mode.', + file=sys.stderr) + sys.exit(1) + # Print known hosts and exit is --list is specified - do not start # the scanner. if args.list: diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index f3976370..459138e2 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -63,6 +63,69 @@ import cdist.config logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") +datetime_format = '%Y-%m-%d %H:%M:%S' + +class Host(object): + def __init__(self, addr, outdir, name_mapper=None): + self.addr = addr + self.workdir = os.path.join(outdir, addr) + self.name_mapper = name_mapper + + os.makedirs(self.workdir, exist_ok=True) + + def __get(self, key, default=None): + fname = os.path.join(self.workdir, key) + value=default + if os.path.isfile(fname): + with open(fname, "r") as fd: + value = fd.readline() + return value + + def __set(self, key, value): + fname = os.path.join(self.workdir, key) + with open(fname, "w") as fd: + fd.write(f"{value}") + + def name(self, default=None): + if self.name_mapper == None: + return default + + fpath = os.path.join(os.getcwd(), self.name_mapper) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: + value = out.stdout.decode() + return (None if len(value) == 0 else value) + else: + return default + + def address(self): + return self.addr + + def last_seen(self, default=None): + raw = self.__get('last_seen') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def last_configured(self, default=None): + raw = self.__get('last_configured') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def seen(self): + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_seen', now) + + def configure(self): + # TODO: configure. + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_configured', now) class Trigger(object): """ @@ -108,48 +171,43 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, args=None, outdir=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): self.interfaces = interfaces + self.autoconfigure=autoconfigure + self.name_mapper = name_mapper + self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') + os.makedirs(self.outdir, exist_ok=True) + + self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = pkg['IPv6'].src - log.verbose("Host %s is alive", host) + host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) + if host.name(): + log.verbose("Host %s (%s) is alive", host.name(), host.address()) + else: + log.verbose("Host %s is alive", host.address()) + host.seen() - dir = os.path.join(self.outdir, host) - fname = os.path.join(dir, "last_seen") - - now = datetime.datetime.now() - - os.makedirs(dir, exist_ok=True) - - # FIXME: maybe adjust the format so we can easily parse again - with open(fname, "w") as fd: - fd.write(f"{now}\n") + # TODO check last config. + if self.autoconfigure and \ + host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + self.config(host) def list(self): - hosts = dict() - for linklocal_addr in os.listdir(self.outdir): - workdir = os.path.join(self.outdir, linklocal_addr) - # We ignore any (unexpected) file in this directory. - if os.path.isdir(workdir): - last_seen='-' - last_seen_file = os.path.join(workdir, 'last_seen') - if os.path.isfile(last_seen_file): - with open(last_seen_file, "r") as fd: - last_seen = fd.readline() - - hosts[linklocal_addr] = {'last_seen': last_seen} + hosts = [] + for addr in os.listdir(self.outdir): + hosts.append(Host(addr, self.outdir, self.name_mapper)) return hosts - def config(self): + def config(self, host): """ Configure a host @@ -158,9 +216,19 @@ class Scanner(object): - Maybe keep dict storing per host processes - Save the result - Save the output -> probably aligned to config mode - """ + if host.name() == None: + log.debug("config - could not resolve name for %s, aborting.", host.address()) + return + + if self.running_configs.get(host.name()) != None: + log.debug("config - is already running for %s, aborting.", host.name()) + + log.info("config - running against host %s.", host.name()) + p = host.configure() + self.running_configs[host.name()] = p + def start(self): self.process = Process(target=self.scan) self.process.start() From 92fff7cb77271009f25d999b6451dc1e43e9bb6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:20 +0200 Subject: [PATCH 008/128] [scanner] fix crash on --list with name mapper provided --- cdist/scan/commandline.py | 10 ++++++++-- cdist/scan/scan.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 1a0ab0a7..3eb7eec4 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -65,11 +65,17 @@ def list(scan, args): 'last configured'.ljust(date_max_size))) print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: + last_seen = host.last_seen() + last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + + last_configured = host.last_configured() + last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), host.address().ljust(ipv6_max_size), - host.last_seen().ljust(date_max_size), - host.last_configured().ljust(date_max_size))) + last_seen.ljust(date_max_size), + last_configured.ljust(date_max_size))) # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 459138e2..69a4121d 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -97,7 +97,7 @@ class Host(object): return default else: value = out.stdout.decode() - return (None if len(value) == 0 else value) + return (default if len(value) == 0 else value) else: return default From 3a9dd5b1669fa29e1713cdadec1596d859d377f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 12:09:55 +0200 Subject: [PATCH 009/128] [scanner] add minimal (non-configurable) config mode --- cdist/scan/scan.py | 35 +++++++++++++++++++---------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 69a4121d..152abb4e 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -122,8 +122,20 @@ class Host(object): now = datetime.datetime.now().strftime(datetime_format) self.__set('last_seen', now) + # XXX: There's no easy way to use the config module without feeding it with + # CLI args. Might as well call everything from scratch! def configure(self): - # TODO: configure. + target = self.name() or self.address() + cmd = ['cdist', 'config', '-v', target ] + + fname = os.path.join(self.workdir, 'last_configuration_log') + with open(fname, "w") as fd: + log.debug("Executing: %s", cmd) + completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) + if completed_process.returncode != 0: + log.error("%s return with non-zero code %i - see %s for details.", + cmd, completed_process.returncode, fname) + now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) @@ -194,7 +206,7 @@ class Scanner(object): log.verbose("Host %s is alive", host.address()) host.seen() - # TODO check last config. + # Configure if needed. if self.autoconfigure and \ host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): self.config(host) @@ -206,27 +218,18 @@ class Scanner(object): return hosts - def config(self, host): - """ - Configure a host - - - Assume we are only called if necessary - - However we need to ensure to not run in parallel - - Maybe keep dict storing per host processes - - Save the result - - Save the output -> probably aligned to config mode - """ - if host.name() == None: log.debug("config - could not resolve name for %s, aborting.", host.address()) return - if self.running_configs.get(host.name()) != None: + previous_config_process = self.running_configs.get(host.name()) + if previous_config_process != None and previous_config_process.is_alive(): log.debug("config - is already running for %s, aborting.", host.name()) - log.info("config - running against host %s.", host.name()) - p = host.configure() + log.info("config - running against host %s (%s).", host.name(), host.address()) + p = Process(target=host.configure()) + p.start() self.running_configs[host.name()] = p def start(self): From 2232435c2206b2b3a9a5f2b976df16540b3f1eeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 26 Apr 2021 14:39:26 +0200 Subject: [PATCH 010/128] [scanner] initial documentation Note: still needs to patch main cdist(1) manpage --- cdist/scan/scan.py | 32 ------------- docs/src/cdist-scan.rst | 99 +++++++++++++++++++++++++++++++++++++++++ docs/src/index.rst | 1 + 3 files changed, 100 insertions(+), 32 deletions(-) create mode 100644 docs/src/cdist-scan.rst diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 152abb4e..2912dab3 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,38 +19,6 @@ # # -# -# Interface to be implemented: -# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated -# scan: scan / listen for icmp6 replies -# trigger: send trigger to multicast -# config: configure newly detected hosts -# install: install newly detected hosts -# -# Scanner logic -# - save results to configdir: -# basedir = ~/.cdist/scan/ -# last_seen = ~/.cdist/scan//last_seen -- record unix time -# or similar -# last_configured = ~/.cdist/scan//last_configured -- record -# unix time or similar -# last_installed = ~/.cdist/scan//last_configured -- record -# unix time or similar -# -# -# -# -# cdist scan --list -# Show all known hosts including last seen flag -# -# Logic for reconfiguration: -# -# - record when configured last time -# - introduce a parameter --reconfigure-after that takes time argument -# - reconfigure if a) host alive and b) reconfigure-after time passed -# - - from multiprocessing import Process import os import logging diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst new file mode 100644 index 00000000..02193456 --- /dev/null +++ b/docs/src/cdist-scan.rst @@ -0,0 +1,99 @@ +Scan +===== + +Description +----------- +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. It is especially useful in netboot-based environment where hosts +boot unconfigured, and to ensure your infrastructure stays in sync with your +configuration. + +This feature is still consider to be in **beta** stage. + +Usage (Examples) +---------------- + +Discover hosts on local network and configure those whose name is resolved by +the name mapper script. + +.. code-block:: sh + + $ cdist scan --beta --interface eth0 \ + --mode scan --name-mapper path/to/script \ + --mode trigger --mode config + +List known hosts and exit. + +.. code-block:: sh + + $ cdist scan --beta --list --name-mapper path/to/script + +Please refer to `cdist(1)` for a detailed list of parameters. + +Modes +----- + +The scanner has 3 modes that can be independently toggled. If the `--mode` +parameter is not specified, only `tigger` and `scan` are enabled (= hosts are +not configured). + +trigger + Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to + trigger detection by the `scan` module. + +scan + Watch for incoming ICMPv6 replies and optionally configure detected hosts. + +config + Enable configuration of hosts detected by `scan`. + +Name Mapper Script +------------------ + +The name mapper script takes an IPv6 address as first argument and writes the +resolved name to stdout - if any. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + case "$1" in + "fe80::20d:b9ff:fe57:3524") + printf "my-host-01" + ;; + "fe80::7603:bdff:fe05:89bb") + printf "my-host-02" + ;; + esac + +Resolving name from `PTR` DNS record: + +.. code-block:: sh + #!/bin/sh + + for cmd in dig sed; do + if ! command -v $cmd > /dev/null; then + exit 1 + fi + done + + dig +short -x "$1" | sed -e 's/.$//' + + +Trigger Source Script +--------------------- + +This script returns a list of addresses (separated by a newline) to be used by +`trigger` mode. It is not used to map names. The script must be executable. + +Simplest script: + +.. code-block:: sh + #!/bin/sh + + cat << EOF + server1.domain.tld + server2.domain.tld + server3.domain.tld + EOF diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..369d5309 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,6 +34,7 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos + cdist-scan cdist-integration cdist-reference cdist-best-practice From a4122882f2a14034d1a219bd0314892176c5cb64 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:39:51 +0200 Subject: [PATCH 011/128] [type/__debconf_set_selections] Add state explorer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit …and to make it work, replace --file with --line. --file is deprecated because it does not work with the state explorer as the contents of the file are not available on the target. --- .../__debconf_set_selections/explorer/state | 124 ++++++++++++++++++ .../__debconf_set_selections/gencode-remote | 37 ++++-- .../type/__debconf_set_selections/man.rst | 49 ++++--- .../type/__debconf_set_selections/manifest | 21 +++ .../parameter/{required => deprecated} | 0 .../parameter/optional_multiple | 1 + 6 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 cdist/conf/type/__debconf_set_selections/explorer/state create mode 100755 cdist/conf/type/__debconf_set_selections/manifest rename cdist/conf/type/__debconf_set_selections/parameter/{required => deprecated} (100%) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/optional_multiple diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state new file mode 100644 index 00000000..68d28941 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -0,0 +1,124 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determine current debconf selections' state. +# Prints one of: +# present: all selections are already set as they should. +# different: one or more of the selections have a different value. +# absent: one or more of the selections are not (currently) defined. +# + +test -x /usr/bin/perl || { + # cannot find perl (no perl ~ no debconf) + echo 'absent' + exit 0 +} + +linesfile="${__object:?}/parameter/line" +test -s "${linesfile}" || { + if test -s "${__object:?}/parameter/file" + then + echo absent + else + echo present + fi + exit 0 +} + +/usr/bin/perl -- - "${linesfile}" <<'EOF' +use strict; +use warnings "all"; + +use Debconf::Db; +use Debconf::Question; + +# Extract @known... arrays from debconf-set-selections +# These values are required to distinguish flags and values in the given lines. +# DC: I couldn't think of a more ugly solution to the problem… +my @knownflags; +my @knowntypes; +my $debconf_set_selections = '/usr/bin/debconf-set-selections'; +if (-e $debconf_set_selections) { + my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; + eval `sed -n '$sed_known' '$debconf_set_selections'`; +} + +sub mungeline ($) { + my $line = shift; + chomp $line; + $line =~ s/\r$//; + return $line; +} + +sub fatal { printf STDERR @_; exit 1; } + +my $state = 'present'; + +sub state { + my $new = shift; + if ($state eq 'present' + or ($state eq 'different' and $new eq 'absent')) { + $state = $new; + } +} + +Debconf::Db->load(readonly => 'true'); + +while (<>) { + # Read and process lines (taken from debconf-set-selections) + $_ = mungeline($_); + while (/\\$/ && ! eof) { + s/\\$//; + $_ .= mungeline(<>); + } + next if /^\s*$/ || /^\s*\#/; + + my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ + or fatal "invalid line: %s\n", $_; + $content = '' unless defined $content; + + + # Compare is and should state + my $q = Debconf::Question->get($label); + + unless (defined $q) { + # probably a preseed + state 'absent'; + next; + } + + if (grep { $_ eq $q->type } @knownflags) { + # This line wants to set a flag, presumably. + if ($q->flag($q->type) ne $content) { + state 'different'; + } + } else { + # Otherwise, it's probably a value… + if ($q->value ne $content) { + state 'different'; + } + + unless (grep { $_ eq $owner } (split /, /, $q->owners)) { + state 'different'; + } + } +} + +printf "%s\n", $state; +EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index e99aef40..50d898c6 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,16 +18,32 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Setup selections -# -filename="$(cat "$__object/parameter/file")" - -if [ "$filename" = "-" ]; then - filename="$__object/stdin" +if test -f "${__object:?}/parameter/line" +then + filename="${__object:?}/parameter/line" +elif test -s "${__object:?}/parameter/file" +then + filename=$(cat "${__object:?}/parameter/file") + if test "${filename}" = '-' + then + filename="${__object:?}/stdin" + fi +else + printf 'Neither --line nor --file set.\n' >&2 + exit 1 fi -echo "debconf-set-selections << __file-eof" -cat "$filename" -echo "__file-eof" +# setting no lines makes no sense +test -s "${filename}" || exit 0 + +state_is=$(cat "${__object:?}/explorer/state") + +if test "${state_is}" != 'present' +then + cat <<-CODE + debconf-set-selections <<'EOF' + $(cat "${filename}") + EOF + CODE +fi diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 58c25b81..690e3e49 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems debconf-set-selections(1) can be used +On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- +cf. ``--line``. + + +OPTIONAL PARAMETERS +------------------- file - Use the given filename as input for debconf-set-selections(1) - If filename is "-", read from stdin. + Use the given filename as input for :strong:`debconf-set-selections`\ (1) + If filename is ``-``, read from stdin. + + **This parameter is deprecated, because it doesn't work with state detection.** +line + A line in :strong:`debconf-set-selections`\ (1) compatible format. + This parameter can be used multiple times to set multiple options. + + (This parameter is actually required, but marked optional because the + deprecated ``--file`` is still accepted.) + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -24,30 +42,29 @@ EXAMPLES .. code-block:: sh - # Setup configuration for nslcd - __debconf_set_selections nslcd --file /path/to/file + # Setup gitolite's gituser + __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' - # Setup configuration for nslcd from another type - __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" - - __debconf_set_selections nslcd --file - << eof - gitolite gitolite/gituser string git - eof + # Setup configuration for nslcd from a file. + # NB: Multiple lines can be passed to --line, although this can be considered a hack. + __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" SEE ALSO -------- -:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`debconf-set-selections`\ (1) AUTHORS ------- Nico Schottelius +Dennis Camera COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__debconf_set_selections/manifest new file mode 100755 index 00000000..0f4fb2e2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/manifest @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +__package_apt debconf diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/required rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple new file mode 100644 index 00000000..a999a0c2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple @@ -0,0 +1 @@ +line From 9cf19388abe95a5f8df9f5c94d2e54fa1060b2ef Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 26 Apr 2021 16:47:44 +0200 Subject: [PATCH 012/128] [type/__debconf_set_selections] Send message about each debconf setting that is changed --- cdist/conf/type/__debconf_set_selections/gencode-remote | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 50d898c6..9ba28f09 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -46,4 +46,9 @@ then $(cat "${filename}") EOF CODE + + awk ' + { + printf "set %s %s %s %s\n", $1, $2, $3, $4 + }' "${filename}" >>"${__messages_out:?}" fi From a42ebc7a78e09e138954b1376376f7cb643420e0 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 27 Apr 2021 19:41:10 +0200 Subject: [PATCH 013/128] [type/__debconf_set_selections] Synchronise objects Works around locking error: debconf: DbDriver "config": /var/cache/debconf/config.dat is locked by another process: Resource temporarily unavailable --- .../__debconf_set_selections/explorer/state | 20 ++++++++++++++++++- .../type/__debconf_set_selections/nonparallel | 0 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__debconf_set_selections/nonparallel diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state index 68d28941..f8a3f6c8 100644 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -41,10 +41,15 @@ test -s "${linesfile}" || { exit 0 } +# assert __type_explorer is set (because it is used by the Perl script) +: "${__type_explorer:?}" + /usr/bin/perl -- - "${linesfile}" <<'EOF' use strict; use warnings "all"; +use Fcntl qw(:DEFAULT :flock); + use Debconf::Db; use Debconf::Question; @@ -78,7 +83,20 @@ sub state { } } -Debconf::Db->load(readonly => 'true'); + +# Load Debconf DB but manually lock on the state explorer script, +# because Debconf aborts immediately if executed concurrently. +# This is not really an ideal solution because the Debconf DB could be locked by +# another process (e.g. apt-get), but no way to achieve this could be found. +# If you know how to, please provide a patch. +my $lockfile = "%ENV{'__type_explorer'}/state"; +if (open my $lock_fh, '+<', $lockfile) { + flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; +} +{ + Debconf::Db->load(readonly => 'true'); +} + while (<>) { # Read and process lines (taken from debconf-set-selections) diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel new file mode 100644 index 00000000..e69de29b From 75c71f69c1fed4371c5891ddcca0eaf28ad928e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 10:17:48 +0200 Subject: [PATCH 014/128] [scanner] pycodestyle compliance --- cdist/scan/commandline.py | 20 +++++++++++---- cdist/scan/scan.py | 54 ++++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 26 deletions(-) diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index 3eb7eec4..b42bc7b2 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -25,13 +25,15 @@ from datetime import datetime log = logging.getLogger("scan") + def run(scan, args): # We run each component in a separate process since they # must not block on each other. processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces, + sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") @@ -48,6 +50,7 @@ def run(scan, args): for process in processes: process.join() + def list(scan, args): s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) hosts = s.list() @@ -66,10 +69,16 @@ def list(scan, args): print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) for host in hosts: last_seen = host.last_seen() - last_seen = last_seen.strftime(scan.datetime_format) if last_seen else '-' + if last_seen: + last_seen = last_seen.strftime(scan.datetime_format) + else: + last_seen = '-' last_configured = host.last_configured() - last_configured = last_configured.strftime(scan.datetime_format) if last_configured else '-' + if last_configured: + last_configured = last_configured.strftime(scan.datetime_format) + else: + '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), @@ -77,6 +86,7 @@ def list(scan, args): last_seen.ljust(date_max_size), last_configured.ljust(date_max_size))) + # CLI processing is defined outside of the main scan class to handle # non-available optional scapy dependency (instead of crashing mid-flight). def commandline(args): @@ -94,9 +104,9 @@ def commandline(args): # By default scan and trigger, but do not call any action. args.mode = ['scan', 'trigger', ] - if 'config' in args.mode and args.name_mapper == None: + if 'config' in args.mode and args.name_mapper is None: print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) + file=sys.stderr) sys.exit(1) # Print known hosts and exit is --list is specified - do not start diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 2912dab3..4a20f511 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -33,6 +33,7 @@ logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") datetime_format = '%Y-%m-%d %H:%M:%S' + class Host(object): def __init__(self, addr, outdir, name_mapper=None): self.addr = addr @@ -43,7 +44,7 @@ class Host(object): def __get(self, key, default=None): fname = os.path.join(self.workdir, key) - value=default + value = default if os.path.isfile(fname): with open(fname, "r") as fd: value = fd.readline() @@ -55,15 +56,15 @@ class Host(object): fd.write(f"{value}") def name(self, default=None): - if self.name_mapper == None: + if self.name_mapper is None: return default fpath = os.path.join(os.getcwd(), self.name_mapper) if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: value = out.stdout.decode() return (default if len(value) == 0 else value) else: @@ -94,19 +95,20 @@ class Host(object): # CLI args. Might as well call everything from scratch! def configure(self): target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target ] + cmd = ['cdist', 'config', '-v', target] fname = os.path.join(self.workdir, 'last_configuration_log') with open(fname, "w") as fd: log.debug("Executing: %s", cmd) completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for details.", - cmd, completed_process.returncode, fname) + log.error("%s return with non-zero code %i - see %s for \ + details.", cmd, completed_process.returncode, fname) now = datetime.datetime.now().strftime(datetime_format) self.__set('last_configured', now) + class Trigger(object): """ Trigger an ICMPv6EchoReply from all hosts that are alive @@ -140,10 +142,12 @@ class Trigger(object): def trigger(self, interface): try: log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6(dst="ff02::1%{}".format(interface)) / ICMPv6EchoRequest() + packet = IPv6( + dst="ff02::1%{}".format(interface) + ) / ICMPv6EchoRequest() send(packet, verbose=self.verbose) except Exception as e: - log.error( "Could not send ICMPv6EchoRequest: %s", e) + log.error("Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -151,9 +155,10 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, name_mapper=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, + name_mapper=None): self.interfaces = interfaces - self.autoconfigure=autoconfigure + self.autoconfigure = autoconfigure self.name_mapper = name_mapper self.config_delay = datetime.timedelta(seconds=3600) @@ -169,14 +174,17 @@ class Scanner(object): if ICMPv6EchoReply in pkg: host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), host.address()) + log.verbose("Host %s (%s) is alive", host.name(), + host.address()) else: log.verbose("Host %s is alive", host.address()) + host.seen() # Configure if needed. if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + self.config_delay < datetime.datetime.now(): + host.last_configured(default=datetime.datetime.min) + \ + self.config_delay < datetime.datetime.now(): self.config(host) def list(self): @@ -187,15 +195,19 @@ class Scanner(object): return hosts def config(self, host): - if host.name() == None: - log.debug("config - could not resolve name for %s, aborting.", host.address()) + if host.name() is None: + log.debug("config - could not resolve name for %s, aborting.", + host.address()) return previous_config_process = self.running_configs.get(host.name()) - if previous_config_process != None and previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", host.name()) + if previous_config_process is not None and \ + previous_config_process.is_alive(): + log.debug("config - is already running for %s, aborting.", + host.name()) - log.info("config - running against host %s (%s).", host.name(), host.address()) + log.info("config - running against host %s (%s).", host.name(), + host.address()) p = Process(target=host.configure()) p.start() self.running_configs[host.name()] = p @@ -214,4 +226,4 @@ class Scanner(object): filter="icmp6", prn=self.handle_pkg) except Exception as e: - log.error( "Could not start listener: %s", e) + log.error("Could not start listener: %s", e) From ab10b453f275d4289f3d37988b3caec204227194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:15:41 +0200 Subject: [PATCH 015/128] [scanner] populate cdist(1) --- docs/src/man1/cdist.rst | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0ecb4a61..599ec3b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,6 +88,9 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] + [-d CONFIG_DELAY] [-t TRIGGER_DELAY] + DESCRIPTION ----------- @@ -641,6 +644,31 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +SCAN +---- + +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. + +**-I INTERFACE, --interfaces INTERFACE** + Interface to listen on. Can be specified multiple times. + +**-m MODE, --mode MODE** + Scanner components to enable. Can be specified multiple time to enable more + than one component. Supported modes are: scan, trigger and config. Defaults + to tiggger and scan. + +**--name-mapper PATH_TO_SCRIPT** + Path to script used to resolve a remote host name from an IPv6 address. + +**--list** + List known hosts and exit. + +**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** + How long (seconds) to wait before reconfiguring after last try (config mode only). + +**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** + How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION ------------- From b8733c65f52776facce7a8de0265538a856ead64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:26:35 +0200 Subject: [PATCH 016/128] [scanner] fix minor CLI handling and --list bugs / typo --- cdist/argparse.py | 4 ++-- cdist/scan/commandline.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index bedb23ac..f17315e7 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -495,7 +495,7 @@ def get_parsers(): action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interfaces', + '-I', '--interface', action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( @@ -503,7 +503,7 @@ def get_parsers(): action='store', default=None, help='Map addresses to names, required for config mode') parser['scan'].add_argument( - '-d', '--delay', + '-d', '--config-delay', action='store', default=3600, type=int, help='How long (seconds) to wait before reconfiguring after last try') parser['scan'].add_argument( diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index b42bc7b2..ddbe4933 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -32,7 +32,7 @@ def run(scan, args): processes = [] if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces, + t = scan.Trigger(interfaces=args.interface, sleeptime=args.trigger_delay) t.start() processes.append(t) @@ -41,7 +41,7 @@ def run(scan, args): if 'scan' in args.mode: s = scan.Scanner( autoconfigure='config' in args.mode, - interfaces=args.interfaces, + interfaces=args.interface, name_mapper=args.name_mapper) s.start() processes.append(s) @@ -52,7 +52,7 @@ def run(scan, args): def list(scan, args): - s = scan.Scanner(interfaces=args.interfaces, name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) hosts = s.list() # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + @@ -75,10 +75,10 @@ def list(scan, args): last_seen = '-' last_configured = host.last_configured() - if last_configured: + if last_configured is not None: last_configured = last_configured.strftime(scan.datetime_format) else: - '-' + last_configured = '-' print("{} | {} | {} | {}".format( host.name(default='-').ljust(name_max_size), From e0c52d0e1dfbaa3814a2ff26482177c2909a15ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 26 May 2021 11:27:11 +0200 Subject: [PATCH 017/128] [scanner] remove mention of non-implemented trigger soruce script --- docs/src/cdist-scan.rst | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 02193456..064e65ff 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -8,7 +8,8 @@ periodically. It is especially useful in netboot-based environment where hosts boot unconfigured, and to ensure your infrastructure stays in sync with your configuration. -This feature is still consider to be in **beta** stage. +This feature is still consider to be in **beta** stage, and only operate on +IPv6 (including link-local). Usage (Examples) ---------------- @@ -79,21 +80,3 @@ Resolving name from `PTR` DNS record: done dig +short -x "$1" | sed -e 's/.$//' - - -Trigger Source Script ---------------------- - -This script returns a list of addresses (separated by a newline) to be used by -`trigger` mode. It is not used to map names. The script must be executable. - -Simplest script: - -.. code-block:: sh - #!/bin/sh - - cat << EOF - server1.domain.tld - server2.domain.tld - server3.domain.tld - EOF From 6ede76b08b96b7504c65ca2dc6737e31be7e7f24 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 8 Jun 2021 16:20:55 +0200 Subject: [PATCH 018/128] [type/__debconf_set_selections] man.rst: Fix line break in AUTHORS --- cdist/conf/type/__debconf_set_selections/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 690e3e49..fd0040ae 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -58,8 +58,8 @@ SEE ALSO AUTHORS ------- -Nico Schottelius -Dennis Camera +| Nico Schottelius +| Dennis Camera COPYING From a3102022e18a23ce9b54eeaf7415b55361f80bd1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:17 +0100 Subject: [PATCH 019/128] More sensible defaults; reword debian-only error message --- cdist/conf/type/__apt_pin/manifest | 13 +++++++++---- cdist/conf/type/__apt_pin/nonparallel | 0 cdist/conf/type/__apt_pin/parameter/default/package | 1 - 3 files changed, 9 insertions(+), 5 deletions(-) create mode 100644 cdist/conf/type/__apt_pin/nonparallel delete mode 100644 cdist/conf/type/__apt_pin/parameter/default/package diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index b1372ad0..909bc80d 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -19,9 +19,17 @@ # +name="$__object_id" + os=$(cat "$__global/explorer/os") state="$(cat "$__object/parameter/state")" -package="$(cat "$__object/parameter/package")" + +if [ -f "$__object/parameter/package" ]; then + package="$(cat "$__object/parameter/package")" +else + package=$name +fi + distribution="$(cat "$__object/parameter/distribution")" priority="$(cat "$__object/parameter/priority")" @@ -31,13 +39,10 @@ case "$os" in ;; *) printf "This type is specific to Debian and it's derivatives" >&2 - printf "If you feel there's an equivalent functionality in %s, please contribute..." "$os" >&2 exit 1 ;; esac -name="$__object_id" - case $distribution in stabletesting|unsatbel|experimental) pin="release a=$distribution" diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_pin/parameter/default/package b/cdist/conf/type/__apt_pin/parameter/default/package deleted file mode 100644 index 72e8ffc0..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/package +++ /dev/null @@ -1 +0,0 @@ -* From b726697e070e5266eae38c7951ced14a2305acb2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 15:05:33 +0100 Subject: [PATCH 020/128] Add documentation --- cdist/conf/type/__apt_pin/man.rst | 37 ++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 7fcae6f8..0c91cdec 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -13,11 +13,21 @@ This space intentionally left blank. REQUIRED PARAMETERS ------------------- -None. +distribution + Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...). OPTIONAL PARAMETERS ------------------- +package + Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + +priority + The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) + +state + Will be passed to underlying `__file` type; see there for valid values and defaults. + None. @@ -31,14 +41,31 @@ EXAMPLES .. code-block:: sh - # TODO - __apt_pin + # Add the bullseye repo to buster, but do not install any pacakges by default + # only if explicitely asked for + __apt_pin bullseye-default \ + --package "*" \ + --distribution bullseye \ + --priority -1 + + require="__apt_pin/bullseye-default" __apt_source bullseye \ + --uri http://deb.debian.org/debian/ \ + --distribution bullseye \ + --component main + # TODO + __apt_pin + + __apt_pin foo --package "foo foo-*" --distribution bullseye + + __foo # Installs the `foo` package internally + + __package foo-plugin-extras SEE ALSO -------- -:strong:`TODO`\ (7) - +:strong:`apt_preferences`\ (7) +:strong:`cdist-type__file`\ (7) AUTHORS ------- From f6b24359d8ebe60bb78de92f626872166156d303 Mon Sep 17 00:00:00 2001 From: pedro Date: Sun, 20 Jun 2021 10:36:09 +0200 Subject: [PATCH 021/128] [type/__download] bugfix checksum not found when the checksum is not found while using __download it displays and error saying "no checksum from target" this information is wrong and confusing. In fact, the state explorer is missing a default condition, when the other conditions are not satisfied --- cdist/conf/type/__download/explorer/state | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 68b517c5..ff984d16 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -61,6 +61,9 @@ else sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" ;; esac + else + echo "could not detect checksum method, check type's manual" >&2 + exit 1 fi fi From 7b3f268df25922c515174862da95569aea547367 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 22 Jun 2021 16:36:30 +0300 Subject: [PATCH 022/128] [__download] improvements 1. post download checksum verification 2. detect hashes without prefix 3. add optional --destination 4. updated man --- .../conf/type/__download/explorer/remote_cmd | 19 --- .../type/__download/explorer/remote_cmd_get | 16 +++ .../type/__download/explorer/remote_cmd_sum | 82 +++++++++++ cdist/conf/type/__download/explorer/state | 60 ++------ cdist/conf/type/__download/gencode-local | 131 +++++++++++++++--- cdist/conf/type/__download/gencode-remote | 46 +++++- cdist/conf/type/__download/man.rst | 31 ++++- cdist/conf/type/__download/manifest | 2 +- cdist/conf/type/__download/parameter/optional | 3 +- 9 files changed, 292 insertions(+), 98 deletions(-) delete mode 100755 cdist/conf/type/__download/explorer/remote_cmd create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_get create mode 100755 cdist/conf/type/__download/explorer/remote_cmd_sum diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd deleted file mode 100755 index e3e35b45..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cmd="$( cat "$__object/parameter/cmd-get" )" - -elif command -v curl > /dev/null -then - cmd="curl -L -o - '%s'" - -elif command -v fetch > /dev/null -then - cmd="fetch -o - '%s'" - -else - cmd="wget -O - '%s'" -fi - -echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get new file mode 100755 index 00000000..9f1cd59c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_get @@ -0,0 +1,16 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cat "$__object/parameter/cmd-get" +elif + command -v curl > /dev/null +then + echo "curl -sSL -o - '%s'" +elif + command -v fetch > /dev/null +then + echo "fetch -o - '%s'" +else + echo "wget -O - '%s'" +fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum new file mode 100755 index 00000000..84df663c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_sum @@ -0,0 +1,82 @@ +#!/bin/sh -e + +if [ ! -f "$__object/parameter/sum" ] +then + exit 0 +fi + +if [ -f "$__object/parameter/cmd-sum" ] +then + cat "$__object/parameter/cmd-sum" + exit 0 +fi + +sum_should="$( cat "$__object/parameter/sum" )" + +if echo "$sum_should" | grep -Fq ':' +then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" +else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi +fi + +os="$( "$__explorer/os" )" + +case "$sum_hash" in + cksum) + echo "cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + case "$os" in + freebsd) + echo "md5 -q %s" + ;; + *) + echo "md5sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha1) + case "$os" in + freebsd) + echo "sha1 -q %s" + ;; + *) + echo "sha1sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha256) + case "$os" in + freebsd) + echo "sha256 -q %s" + ;; + *) + echo "sha256sum %s | awk '{print \$1}'" + ;; + esac + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 68b517c5..881a1c09 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,6 +1,11 @@ #!/bin/sh -e -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ ! -f "$dst" ] then @@ -16,57 +21,18 @@ fi sum_should="$( cat "$__object/parameter/sum" )" -if [ -f "$__object/parameter/cmd-sum" ] +if echo "$sum_should" | grep -Fq ':' then - # shellcheck disable=SC2059 - sum_is="$( eval "$( printf \ - "$( cat "$__object/parameter/cmd-sum" )" \ - "$dst" )" )" -else - os="$( "$__explorer/os" )" - - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" - - elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' - then - case "$os" in - freebsd) - sum_is="md5:$( md5 -q "$dst" )" - ;; - *) - sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' - then - case "$os" in - freebsd) - sum_is="sha1:$( sha1 -q "$dst" )" - ;; - *) - sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' - then - case "$os" in - freebsd) - sum_is="sha256:$( sha256 -q "$dst" )" - ;; - *) - sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" - ;; - esac - fi + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" fi +sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" + +sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" + if [ -z "$sum_is" ] then - echo 'no checksum from target' >&2 + echo 'existing destination checksum failed' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 571d2c3c..d1b0d0d5 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,34 +11,133 @@ fi url="$( cat "$__object/parameter/url" )" -tmp="$( mktemp )" - -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - elif command -v curl > /dev/null then - cmd="curl -L -o - '%s'" + cmd="curl -sSL -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + else - echo 'no usable locally installed utility for downloading' >&2 + echo 'local download failed, no usable utility' >&2 exit 1 fi -printf "$cmd > %s\n" \ - "$url" \ - "$tmp" +echo "download_tmp=\"\$( mktemp )\"" + +# shellcheck disable=SC2059 +printf "$cmd > \"\$download_tmp\"\n" "$url" + +if [ -f "$__object/parameter/sum" ] +then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + if echo "$sum_should" | grep -Fq ':' + then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" + + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi + fi + + case "$sum_hash" in + cksum) + local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + if command -v md5 > /dev/null + then + local_cmd_sum="md5 -q %s" + elif + command -v md5sum > /dev/null + then + local_cmd_sum="md5sum %s | awk '{print \$1}'" + fi + ;; + sha1) + if command -v sha1 > /dev/null + then + local_cmd_sum="sha1 -q %s" + elif + command -v sha1sum > /dev/null + then + local_cmd_sum="sha1sum %s | awk '{print \$1}'" + fi + ;; + sha256) + if command -v sha256 > /dev/null + then + local_cmd_sum="sha256 -q %s" + elif + command -v sha256sum > /dev/null + then + local_cmd_sum="sha256sum %s | awk '{print \$1}'" + fi + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; + esac + + if [ -z "$local_cmd_sum" ] + then + echo 'local checksum verification failed, no usable utility' >&2 + exit 1 + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'local download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' +fi if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -47,12 +146,10 @@ else target_host="$__target_host" fi -printf '%s %s %s:%s\n' \ +# shellcheck disable=SC2016 +printf '%s "$download_tmp" %s:%s\n' \ "$__remote_copy" \ - "$tmp" \ "$target_host" \ "$dst" -echo "rm -f '$tmp'" - -echo 'downloaded' > "$__messages_out" +echo "rm -f \"\$download_tmp\"" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index 029a0801..e49bcec3 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd="$( cat "$__object/explorer/remote_cmd" )" + cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" url="$( cat "$__object/parameter/url" )" - dst="/$__object_id" + if [ -f "$__object/parameter/destination" ] + then + dst="$( cat "$__object/parameter/destination" )" + else + dst="/$__object_id" + fi - printf "$cmd > %s\n" \ - "$url" \ - "$dst" + echo "download_tmp=\"\$( mktemp )\"" - echo 'downloaded' > "$__messages_out" + # shellcheck disable=SC2059 + printf "$cmd_get > \"\$download_tmp\"\n" "$url" + + if [ -f "$__object/parameter/sum" ] + then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" + + if echo "$sum_should" | grep -Fq ':' + then + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'remote download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' + fi + + echo "mv \"\$download_tmp\" '$dst'" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index a1278cfb..c16510a9 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,7 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``wget``, ``curl`` or ``fetch``. +By default type will try to use ``curl``, ``fetch`` or ``wget``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,6 +16,8 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). +To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. + REQUIRED PARAMETERS ------------------- @@ -25,14 +27,29 @@ url OPTIONAL PARAMETERS ------------------- +destination + Downloaded file's destination in target. If unset, ``$__object_id`` is used. + sum - Checksum is used to decide if existing destination file must be redownloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. + + Type tries to detect hash format with regexes, but prefixes + ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. + + Checksum have two purposes - state check and post-download verification. + In state check, if destination checksum mismatches, then content of URL + will be downloaded to temporary file. If downloaded temporary file's + checksum matches, then it will be moved to destination (overwritten). + + For local downloads it is expected that usable utilities for checksum + calculation exist in the system. download - If ``local`` (default), then download file to local storage and copy - it to target host. If ``remote``, then download happens in target. + If ``local`` (default), then file is downloaded to local storage and copied + to target host. If ``remote``, then download happens in target. + + For local downloads it is expected that usable utilities for downloading + exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. cmd-get Command used for downloading. @@ -62,7 +79,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum md5:46da3021ca9eace277115ec9106c5b46 + --sum 46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 7ec8d86d..3d4c498b 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd" +if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index d69e083e..e809ef78 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,5 +1,6 @@ -sum cmd-get cmd-sum +destination download onchange +sum From 2db40d8d704b427768307fbea29384bd3dc8dbd7 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 28 Jun 2021 12:54:20 +0200 Subject: [PATCH 023/128] Use $__remote_exec and thus the ssh multiplexing --- cdist/conf/type/__rsync/gencode-local | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e36ded2f..36addc36 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,4 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ + -e "$__remote_exec" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From d937d53f3dfd10830a07aee0450596eea62f2a1a Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 28 Jun 2021 18:09:35 +0100 Subject: [PATCH 024/128] Add quotes to rsync command --- cdist/conf/type/__rsync/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 36addc36..f1bddc16 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -36,5 +36,5 @@ fi echo rsync -a \ --no-owner --no-group \ - -e "$__remote_exec" \ + -e \"$__remote_exec\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 60753ddfcc3ac49303d572da7f6a68f398c02227 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:42:10 +0300 Subject: [PATCH 025/128] fix shellcheck --- cdist/conf/type/__download/explorer/state | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 881a1c09..8c5d5ce1 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -28,6 +28,7 @@ fi sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" +# shellcheck disable=SC2059 sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" if [ -z "$sum_is" ] From a90e642c1354cf01be7c9a1ba8a468ad624c4202 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 14:50:40 +0300 Subject: [PATCH 026/128] update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index de6901c7..a468dd86 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ freenode +IRC: ``#cdist`` @ [libera](https://libera.chat) Matrix: ``#cdist:ungleich.ch`` -Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist +Matrix and IRC are bridged. From 243a4b904a2de638d00bac57f6e762a076f9ae54 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 2 Jul 2021 06:50:02 +0200 Subject: [PATCH 027/128] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 97ea204b..d48c657e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,8 @@ next: * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) * Type __pyvenv: Fix group explorer (Dennis Camera) + * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) + * Type __debconf_set_selections: Add state explorer (Dennis Camera) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 30ba796d060fd3d0c4affd4167f14784156b5354 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 1 Jul 2021 11:49:07 +0300 Subject: [PATCH 028/128] new type: __snakeoil_cert --- .../__snakeoil_cert/explorer/ssl-cert-group | 8 ++ .../conf/type/__snakeoil_cert/explorer/state | 24 ++++++ .../conf/type/__snakeoil_cert/gencode-remote | 73 +++++++++++++++++++ cdist/conf/type/__snakeoil_cert/man.rst | 60 +++++++++++++++ .../parameter/default/cert-path | 1 + .../parameter/default/key-path | 1 + .../parameter/default/key-type | 1 + .../type/__snakeoil_cert/parameter/optional | 4 + 8 files changed, 172 insertions(+) create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group create mode 100755 cdist/conf/type/__snakeoil_cert/explorer/state create mode 100755 cdist/conf/type/__snakeoil_cert/gencode-remote create mode 100644 cdist/conf/type/__snakeoil_cert/man.rst create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/cert-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-path create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/default/key-type create mode 100644 cdist/conf/type/__snakeoil_cert/parameter/optional diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group new file mode 100755 index 00000000..a6cb3dfd --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group @@ -0,0 +1,8 @@ +#!/bin/sh -e + +if grep -Eq '^ssl-cert:' /etc/group +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state new file mode 100755 index 00000000..cc5aae0b --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/state @@ -0,0 +1,24 @@ +#!/bin/sh -e + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] +then + echo 'absent' +else + echo 'present' +fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote new file mode 100755 index 00000000..8ffbfad1 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e + +state="$( cat "$__object/explorer/state" )" + +if [ "$state" = 'present' ] +then + exit 0 +fi + +if [ -f "$__object/parameter/common-name" ] +then + common_name="$( cat "$__object/parameter/common-name" )" +else + common_name="$__object_id" +fi + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +key_type="$( cat "$__object/parameter/key-type" )" + +key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" + +case "$key_type" in + rsa:*) + echo "openssl genrsa -out '$key_path' $key_type_arg" + ;; + ec:*) + echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" + ;; +esac + +# shellcheck disable=SC2016 +echo 'csr_path="$( mktemp )"' + +echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" + +echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" + +# shellcheck disable=SC2016 +echo 'rm -f "$csr_path"' + +if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] +then + key_group='ssl-cert' +else + key_group='root' +fi + +echo "chmod 640 '$key_path'" + +echo "chown root '$key_path'" + +echo "chgrp $key_group '$key_path'" + +echo "chmod 644 '$cert_path'" + +echo "chown root '$cert_path'" + +echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst new file mode 100644 index 00000000..0b547804 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -0,0 +1,60 @@ +cdist-type__snakeoil_cert(7) +============================ + +NAME +---- +cdist-type__snakeoil_cert - Generate self-signed certificate + + +DESCRIPTION +----------- +The purpose of this type is to generate **self-signed** certificate and private key +for **testing purposes**. Certificate will expire in 3650 days. + +Certificate's and key's access bits will be ``644`` and ``640`` respectively. +If target system has ``ssl-cert`` group, then it will be used as key's group. +Use ``require='__snakeoil_cert/...' __file ...`` to override. + + +OPTIONAL PARAMETERS +------------------- +common-name + Defaults to ``$__object_id``. + +key-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/private/%s.pem``. + +key-type + Possible values are ``rsa:$bits`` and ``ec:$name``. + For possible EC names see ``openssl ecparam -list_curves``. + Defaults to ``rsa:2048``. + +cert-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/certs/%s.pem``. + + +EXAMPLES +-------- +.. code-block:: sh + __snakeoil_cert localhost-rsa \ + --common-name localhost \ + --key-type rsa:4096 + + __snakeoil_cert localhost-ec \ + --common-name localhost \ + --key-type ec:prime256v1 + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path new file mode 100644 index 00000000..4bbae089 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path @@ -0,0 +1 @@ +/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path new file mode 100644 index 00000000..86eb9359 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path @@ -0,0 +1 @@ +/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type new file mode 100644 index 00000000..f13f8ada --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type @@ -0,0 +1 @@ +rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional new file mode 100644 index 00000000..76d08c0a --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/optional @@ -0,0 +1,4 @@ +common-name +key-path +key-type +cert-path From 853e5cf7b4f615e1470c7ec2cfa8fe68cfa85ce7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 5 Jul 2021 09:07:06 +0200 Subject: [PATCH 029/128] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index d48c657e..0c3c64e1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,8 @@ next: * Type __pyvenv: Fix group explorer (Dennis Camera) * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) * Type __debconf_set_selections: Add state explorer (Dennis Camera) + * Core: Implement usable cdist scan (Timothée Floure) + * New type: __snakeoil_cert (Ander Punnar) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From be92731c5c8a8543448f0d87fafae67e22ac76a1 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Mon, 5 Jul 2021 12:38:26 +0100 Subject: [PATCH 030/128] Shell check quoting We're actually echo-ing the command, hence the escape in front of the quotes - the issue Shellcheck alludes too would actually occur, had the escaping bakcslashes been omitted. --- cdist/conf/type/__rsync/gencode-local | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index f1bddc16..be4feabb 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -34,7 +34,8 @@ if [ -f "$__object/parameter/rsync-opts" ]; then done < "$__object/parameter/rsync-opts" fi +# shellcheck disable=SC2086 echo rsync -a \ --no-owner --no-group \ - -e \"$__remote_exec\" \ + -e \"${__remote_exec}\" \ -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" From 521241d74102e37fae5f55552e1ef565d26ea9d2 Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:28:05 +0200 Subject: [PATCH 031/128] Refine docs even more --- cdist/conf/type/__apt_pin/man.rst | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 0c91cdec..4229c0cd 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -3,12 +3,12 @@ cdist-type__apt_pin(7) NAME ---- -cdist-type__apt_pin - TODO +cdist-type__apt_pin - Manage apt pinning rules DESCRIPTION ----------- -This space intentionally left blank. +Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.) REQUIRED PARAMETERS @@ -20,7 +20,7 @@ distribution OPTIONAL PARAMETERS ------------------- package - Package name or glob/RE expression to match multiple packages. If not specified `__object_id` is used. + Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) @@ -28,7 +28,6 @@ priority state Will be passed to underlying `__file` type; see there for valid values and defaults. -None. BOOLEAN PARAMETERS @@ -41,8 +40,8 @@ EXAMPLES .. code-block:: sh - # Add the bullseye repo to buster, but do not install any pacakges by default - # only if explicitely asked for + # Add the bullseye repo to buster, but do not install any packages by default, + # only if explicitely asked for (-1 means "never" for apt) __apt_pin bullseye-default \ --package "*" \ --distribution bullseye \ @@ -52,19 +51,19 @@ EXAMPLES --uri http://deb.debian.org/debian/ \ --distribution bullseye \ --component main - # TODO - __apt_pin __apt_pin foo --package "foo foo-*" --distribution bullseye - __foo # Installs the `foo` package internally + __foo # Assuming, this installs the `foo` package internally - __package foo-plugin-extras + __package foo-plugin-extras # Assuming we also need some extra stuff SEE ALSO -------- -:strong:`apt_preferences`\ (7) +:strong:`apt_preferences`\ (5) +:strong:`cdist-type__apt_source`\ (7) +:strong:`cdist-type__apt_backports`\ (7) :strong:`cdist-type__file`\ (7) AUTHORS From 166b58aeea09da41086525849291c70a3c3a571c Mon Sep 17 00:00:00 2001 From: fancsali Date: Mon, 5 Jul 2021 15:32:27 +0200 Subject: [PATCH 032/128] Fix typo in distro names... --- cdist/conf/type/__apt_pin/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 909bc80d..e72a8fdd 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -44,7 +44,7 @@ case "$os" in esac case $distribution in - stabletesting|unsatbel|experimental) + stable|testing|unstable|experimental) pin="release a=$distribution" ;; *) From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 20:47:22 +0300 Subject: [PATCH 033/128] new type: __sed --- cdist/conf/type/__sed/explorer/file | 22 +++++++++ cdist/conf/type/__sed/gencode-remote | 46 +++++++++++++++++ cdist/conf/type/__sed/man.rst | 49 +++++++++++++++++++ cdist/conf/type/__sed/parameter/boolean | 1 + cdist/conf/type/__sed/parameter/optional | 1 + .../type/__sed/parameter/required_multiple | 1 + 6 files changed, 120 insertions(+) create mode 100755 cdist/conf/type/__sed/explorer/file create mode 100755 cdist/conf/type/__sed/gencode-remote create mode 100644 cdist/conf/type/__sed/man.rst create mode 100644 cdist/conf/type/__sed/parameter/boolean create mode 100644 cdist/conf/type/__sed/parameter/optional create mode 100644 cdist/conf/type/__sed/parameter/required_multiple diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file new file mode 100755 index 00000000..a40f71b3 --- /dev/null +++ b/cdist/conf/type/__sed/explorer/file @@ -0,0 +1,22 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +if [ -f "$file" ] +then + if [ -s "$file" ] + then + cat "$file" + else + echo "$file is empty" >&2 + exit 1 + fi +else + echo "$file do not exist" >&2 + exit 1 +fi diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote new file mode 100755 index 00000000..391e3bc1 --- /dev/null +++ b/cdist/conf/type/__sed/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +script="$( cat "$__object/parameter/script" )" + +if [ "$script" = '-' ] +then + script="$( cat "$__object/stdin" )" +elif + [ -f "$script" ] +then + script="$( cat "$script" )" +fi + +file_from_target="$__object/explorer/file" + +sed_cmd='sed' + +if [ -f "$__object/parameter/regexp-extended" ] +then + sed_cmd="$sed_cmd --regexp-extended" +fi + +if ! echo "$script" \ + | "$sed_cmd" -f - "$file_from_target" \ + | diff "$file_from_target" - \ + > /dev/null +then + echo 'tmp="$( mktemp )"' + + echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + + echo "$script" + + echo 'EOF' + + echo "cp \"\$tmp\" '$file'" + + echo 'rm -f "$tmp"' +fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst new file mode 100644 index 00000000..fbac212d --- /dev/null +++ b/cdist/conf/type/__sed/man.rst @@ -0,0 +1,49 @@ +cdist-type__sed(7) +================== + +NAME +---- +cdist-type__sed - Transform text files with ``sed`` + + +DESCRIPTION +----------- +TODO + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +script + TODO + + +OPTIONAL PARAMETERS +------------------- +file + TODO + + +BOOLEAN PARAMETERS +------------------ +regexp-extended + TODO + + +EXAMPLES +-------- +.. code-block:: sh + + true + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean new file mode 100644 index 00000000..1ad75c5d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/boolean @@ -0,0 +1 @@ +regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional new file mode 100644 index 00000000..f73f3093 --- /dev/null +++ b/cdist/conf/type/__sed/parameter/optional @@ -0,0 +1 @@ +file diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple new file mode 100644 index 00000000..84f7e31d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/required_multiple @@ -0,0 +1 @@ +script From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:23:25 +0300 Subject: [PATCH 034/128] add --onchange, fix shellcheck --- cdist/conf/type/__sed/gencode-remote | 9 ++++++++- cdist/conf/type/__sed/man.rst | 3 +++ cdist/conf/type/__sed/parameter/optional | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 391e3bc1..6b1bd216 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -32,6 +32,7 @@ if ! echo "$script" \ | diff "$file_from_target" - \ > /dev/null then + # shellcheck disable=SC2016 echo 'tmp="$( mktemp )"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" @@ -41,6 +42,12 @@ then echo 'EOF' echo "cp \"\$tmp\" '$file'" - + + # shellcheck disable=SC2016 echo 'rm -f "$tmp"' + + if [ -f "$__object/parameter/onchange" ] + then + cat "$__object/parameter/onchange" + fi fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index fbac212d..4a728184 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -22,6 +22,9 @@ OPTIONAL PARAMETERS file TODO +onchange + TODO + BOOLEAN PARAMETERS ------------------ diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional index f73f3093..fa86f917 100644 --- a/cdist/conf/type/__sed/parameter/optional +++ b/cdist/conf/type/__sed/parameter/optional @@ -1 +1,2 @@ file +onchange From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:28:00 +0300 Subject: [PATCH 035/128] add messaging and exit earlier --- cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 6b1bd216..836506f0 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -27,27 +27,31 @@ then sed_cmd="$sed_cmd --regexp-extended" fi -if ! echo "$script" \ +if echo "$script" \ | "$sed_cmd" -f - "$file_from_target" \ | diff "$file_from_target" - \ > /dev/null then - # shellcheck disable=SC2016 - echo 'tmp="$( mktemp )"' - - echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - - echo "$script" - - echo 'EOF' - - echo "cp \"\$tmp\" '$file'" - - # shellcheck disable=SC2016 - echo 'rm -f "$tmp"' - - if [ -f "$__object/parameter/onchange" ] - then - cat "$__object/parameter/onchange" - fi + exit 0 +fi + +# shellcheck disable=SC2016 +echo 'tmp="$( mktemp )"' + +echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + +echo "$script" + +echo 'EOF' + +echo "cp \"\$tmp\" '$file'" + +# shellcheck disable=SC2016 +echo 'rm -f "$tmp"' + +echo 'change' >> "$__messages_out" + +if [ -f "$__object/parameter/onchange" ] +then + cat "$__object/parameter/onchange" fi From 3e76d1cd3fbe53fa77c460cb2ce8416698b101bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Jul 2021 08:09:05 +0200 Subject: [PATCH 036/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0c3c64e1..04f826f0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -13,6 +13,7 @@ next: * Type __debconf_set_selections: Add state explorer (Dennis Camera) * Core: Implement usable cdist scan (Timothée Floure) * New type: __snakeoil_cert (Ander Punnar) + * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) From 77dab4c5c63070aef962875af7fe8b1565f5ba78 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 20:37:02 +0200 Subject: [PATCH 037/128] Release 6.9.7 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 04f826f0..284293c1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) From 65c43d3c1db938ac0063f54b1cb6b5090bb0a665 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 10 Jul 2021 21:02:27 +0200 Subject: [PATCH 038/128] Fix docs code block errors --- cdist/conf/type/__snakeoil_cert/man.rst | 1 + docs/src/cdist-scan.rst | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst index 0b547804..b0b0a2e9 100644 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -38,6 +38,7 @@ cert-path EXAMPLES -------- .. code-block:: sh + __snakeoil_cert localhost-rsa \ --common-name localhost \ --key-type rsa:4096 diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst index 064e65ff..86b7fab6 100644 --- a/docs/src/cdist-scan.rst +++ b/docs/src/cdist-scan.rst @@ -57,6 +57,7 @@ resolved name to stdout - if any. The script must be executable. Simplest script: .. code-block:: sh + #!/bin/sh case "$1" in @@ -71,6 +72,7 @@ Simplest script: Resolving name from `PTR` DNS record: .. code-block:: sh + #!/bin/sh for cmd in dig sed; do From 0e611af2a6388572eef5112c2ffaed082803965c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 13 Jul 2021 00:13:22 +0300 Subject: [PATCH 039/128] [__rsync] rewrite --- cdist/conf/type/__rsync/gencode-local | 123 +++++++++++++----- cdist/conf/type/__rsync/gencode-remote | 37 ------ cdist/conf/type/__rsync/man.rst | 107 +++++---------- cdist/conf/type/__rsync/manifest | 18 --- .../type/__rsync/parameter/default/options | 1 + cdist/conf/type/__rsync/parameter/optional | 4 +- .../type/__rsync/parameter/optional_multiple | 2 +- 7 files changed, 130 insertions(+), 162 deletions(-) delete mode 100755 cdist/conf/type/__rsync/gencode-remote create mode 100644 cdist/conf/type/__rsync/parameter/default/options diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index be4feabb..612d237e 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,41 +1,100 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -source=$(cat "$__object/parameter/source") -remote_user=$(cat "$__object/parameter/remote-user") +if ! command -v rsync > /dev/null +then + echo 'rsync is missing in local machine' >&2 + exit 1 +fi -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") +src="$( cat "$__object/parameter/source" )" + +if [ ! -e "$src" ] +then + echo "$src not found" >&2 + exit 1 +fi + +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" else - destination="/$__object_id" + dst="/$__object_id" fi -set -- -if [ -f "$__object/parameter/rsync-opts" ]; then - while read -r opts; do - set -- "$@" "--$opts" - done < "$__object/parameter/rsync-opts" +# if source is directory, then make sure that +# source and destination are ending with slash, +# because this is what you almost always want when +# rsyncing two directories. + +if [ -d "$src" ] +then + if ! echo "$src" | grep -Eq '/$' + then + src="$src/" + fi + + if ! echo "$dst" | grep -Eq '/$' + then + dst="$dst/" + fi fi +remote_user="$( cat "$__object/parameter/remote-user" )" + +options="$( cat "$__object/parameter/options" )" + +if [ -f "$__object/parameter/option" ] +then + while read -r l + do + # there's a limitation in argparse: value can't begin with '-'. + # to workaround this, let's prefix opts with '\' in manifest and remove here. + # read more about argparse issue: https://bugs.python.org/issue9334 + + options="$options $( echo "$l" | sed 's/\\//g' )" + done \ + < "$__object/parameter/option" +fi + +if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] +then + options="$options --chown=" + + if [ -f "$__object/parameter/owner" ] + then + owner="$( cat "$__object/parameter/owner" )" + options="$options$owner" + fi + + if [ -f "$__object/parameter/group" ] + then + group="$( cat "$__object/parameter/group" )" + options="$options:$group" + fi +fi + +if [ -f "$__object/parameter/mode" ] +then + mode="$( cat "$__object/parameter/mode" )" + options="$options --chmod=$mode" +fi + +# IMPORTANT +# +# 1. we first dry-run rsync with change summary to find out +# if there are any changes and code generation is needed. +# 2. normally, to get current state or target host, we run +# such operations in type explorers, but that's not +# possible due to how rsync works. +# 3. redirecting output of dry-run to stderr to ease debugging. +# 4. to understand how that cryptic regex works, please +# open rsync manpage and read about --itemize-changes. + # shellcheck disable=SC2086 -echo rsync -a \ - --no-owner --no-group \ - -e \"${__remote_exec}\" \ - -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" +if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ + | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 +then + exit 0 +fi + +echo "rsync $options $src $remote_user@$__target_host:$dst" diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote deleted file mode 100755 index 074246af..00000000 --- a/cdist/conf/type/__rsync/gencode-remote +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") -else - destination="/$__object_id" -fi - -ownergroup="" -if [ -f "$__object/parameter/owner" ]; then - ownergroup=$(cat "$__object/parameter/owner") -fi -if [ -f "$__object/parameter/group" ]; then - ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" -fi - -if [ "$ownergroup" ]; then - echo chown -R "$ownergroup" "$destination" -fi diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 94b06d63..88019c92 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,112 +3,73 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using rsync +cdist-type__rsync - Mirror directories using ``rsync`` DESCRIPTION ----------- -WARNING: This type is of BETA quality: - -- it has not been tested widely -- interfaces *may* change -- if there is a better approach to solve the problem -> the type may even vanish - -If you are fine with these constraints, please read on. - - -This cdist type allows you to mirror local directories to the -target host using rsync. Rsync will be installed in the manifest of the type. -If group or owner are giveng, a recursive chown will be executed on the -target host. - -A slash will be appended to the source directory so that only the contents -of the directory are taken and not the directory name itself. +The purpose of this type is to bring power of ``rsync`` into ``cdist``. REQUIRED PARAMETERS ------------------- source - Where to take files from + Source directory in local machine. + If source is directory, slash (``/``) will be added to source and destination paths. OPTIONAL PARAMETERS ------------------- -group - Group to chgrp to. +destination + Destination directory. Defaults to ``$__object_id``. owner - User to chown to. + Will be passed to ``rsync`` as ``--chown=OWNER``. + Read ``rsync(1)`` for more details. -destination - Use this as the base destination instead of the object id +group + Will be passed to ``rsync`` as ``--chown=:GROUP``. + Read ``rsync(1)`` for more details. + +mode + Will be passed to ``rsync`` as ``--chmod=MODE``. + Read ``rsync(1)`` for more details. + +options + Defaults to ``--recursive --links --perms --times``. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. remote-user - Use this user instead of the default "root" for rsync operations. + Defaults to ``root``. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -rsync-opts - Use this option to give rsync options with. - See rsync(1) for available options. - Only "--" options are supported. - Write the options without the beginning "--" - Can be specified multiple times. - - -MESSAGES --------- -NONE +option + Pass additional options to ``rsync``. + See ``rsync(1)`` for all possible options. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. EXAMPLES -------- - .. code-block:: sh - # You can use any source directory - __rsync /tmp/testdir \ - --source /etc - - # Use source from type - __rsync /etc \ - --source "$__type/files/package" - - # Allow multiple __rsync objects to write to the same dir - __rsync mystuff \ - --destination /usr/local/bin \ - --source "$__type/files/package" - - __rsync otherstuff \ - --destination /usr/local/bin \ - --source "$__type/files/package2" - - # Use rsync option --exclude - __rsync /tmp/testdir \ - --source /etc \ - --rsync-opts exclude=sshd_conf - - # Use rsync with multiple options --exclude --dry-run - __rsync /tmp/testing \ - --source /home/tester \ - --rsync-opts exclude=id_rsa \ - --rsync-opts dry-run - - -SEE ALSO --------- -:strong:`rsync`\ (1) + __rsync /var/www/example.com \ + --owner root \ + --group www-data \ + --mode 'D750,F640' \ + --source "$__files/example.com/www" AUTHORS ------- -Nico Schottelius +Ander Punnar COPYING ------- -Copyright \(C) 2015 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 9bd44c6d..64fa804e 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,21 +1,3 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options new file mode 100644 index 00000000..d967b110 --- /dev/null +++ b/cdist/conf/type/__rsync/parameter/default/options @@ -0,0 +1 @@ +--recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index ac2b2390..833e9bbe 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,4 +1,6 @@ destination -owner group +mode +options +owner remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index fdb7cd88..01925a15 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -rsync-opts +option From 46b5c24cd240cd9006d93cfcc12a4d81b46a5238 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Sun, 18 Jul 2021 16:25:00 +0300 Subject: [PATCH 040/128] use $__remote_exec for RSYNC_RSH --- cdist/conf/type/__rsync/gencode-local | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index 612d237e..e9f3c131 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -90,6 +90,8 @@ fi # 4. to understand how that cryptic regex works, please # open rsync manpage and read about --itemize-changes. +export RSYNC_RSH="$__remote_exec" + # shellcheck disable=SC2086 if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 @@ -97,4 +99,6 @@ then exit 0 fi +echo "export RSYNC_RSH='$__remote_exec'" + echo "rsync $options $src $remote_user@$__target_host:$dst" From 5229337611b7e7afb2597729a786c637a8bec1f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:41:29 +0200 Subject: [PATCH 041/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 284293c1..f0746218 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __rsync: Rewrite (Ander Punnar) + 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) From de116661613002e91b54194ac8f0d5c3dd3eebbd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 18 Jul 2021 17:45:19 +0200 Subject: [PATCH 042/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f0746218..ea51ac1b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) + * New type: __apt_pin (Daniel Fancsali) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 24c9406ea0f2c6c8edc87e5bbc1be25b5e8e1572 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 19 Jul 2021 12:13:23 +0200 Subject: [PATCH 043/128] [explorer/os_version] Convert Devuan ceres to version number Conversion of Devuan ceres to version numbers is done based on Devuan codenames. The version number is the version number of the final release - 0.01. Analogous to Debian. --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..6c94915c 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,12 +18,11 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# # All os variables are lower case # -# -case "$("$__explorer/os")" in +case $("${__explorer:?}/os") +in amazon) cat /etc/system-release ;; @@ -59,7 +59,23 @@ case "$("$__explorer/os")" in esac ;; devuan) - cat /etc/devuan_version + devuan_version=$(cat /etc/devuan_version) + case ${devuan_version} + in + (*/ceres) + # ceres versions don't have a number, so we decode by codename: + case ${devuan_version} + in + (chimaera/ceres) echo 3.99 ;; + (beowulf/ceres) echo 2.99 ;; + (ascii/ceres) echo 1.99 ;; + (*) exit 1 + esac + ;; + (*) + echo "${devuan_version}" + ;; + esac ;; fedora) cat /etc/fedora-release From fbc9594729ece74b2f8612fe8046ddea966e8c05 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Jul 2021 06:38:46 +0200 Subject: [PATCH 044/128] ++changelog --- docs/changelog | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/changelog b/docs/changelog index ea51ac1b..55d7cb73 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) + * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) @@ -146,7 +147,7 @@ next: * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -211,7 +212,7 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1070,9 +1071,9 @@ next: * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 From c7daaabc6c28cbfaebaf6c620cc3cac130684d2c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 20 Jul 2021 09:03:16 +0200 Subject: [PATCH 045/128] [docs] Bump copyright year to 2021 --- docs/src/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/conf.py b/docs/src/conf.py index 47765413..a3dfafca 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2020' +copyright = 'ungleich GmbH 2021' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for From fed01ded83b217a3ad147f2403ca863be373eeaa Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Thu, 22 Jul 2021 11:17:41 +0200 Subject: [PATCH 046/128] [cdist.log] Define custom log functions on logging.Logger Define out custom logger functions on logging.Logger so that they are passed on to all other loggers. Also, the logger functions need to take a self argument so that they can log on the corrent Logger. --- cdist/log.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index 113f3b4c..62e457fe 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,25 +36,27 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') + logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(msg, *args, **kwargs): - logging.log(logging.VERBOSE, msg, *args, **kwargs) +def _verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, args, **kwargs) -logging.verbose = _verbose +logging.Logger.verbose = _verbose + logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(msg, *args, **kwargs): - logging.log(logging.TRACE, msg, *args, **kwargs) +def _trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) -logging.trace = _trace +logging.Logger.trace = _trace class CdistFormatter(logging.Formatter): From 71fee1fd6b4874abffee5173a96c31842a9583bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 23 Jul 2021 08:06:45 +0200 Subject: [PATCH 047/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 55d7cb73..1273e432 100644 --- a/docs/changelog +++ b/docs/changelog @@ -5,6 +5,7 @@ next: * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) + * Core: Fix logging bug (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 67bcc6cae38e2d480c3336b6a4d59d01196190e3 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sat, 24 Jul 2021 02:37:58 +0200 Subject: [PATCH 048/128] Improve Makefile compatibility and build docs We now use `$(MAKE)` for subsequent calls to `make`. This means that systems that do not default to GNU make can run `gmake man` and produce the man pages. While there also document a dependency on the rtd theme for sphinx. --- Makefile | 6 +++--- docs/src/cdist-install.rst | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 3712511c..89286310 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=make -C $(DOCS_SRC_DIR) man -SPHINXH=make -C $(DOCS_SRC_DIR) html -SPHINXC=make -C $(DOCS_SRC_DIR) clean +SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man +SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html +SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 18863145..390ab9ec 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx (for building html docs and/or the man pages) + * sphinx with the rtd theme (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ From cb8695cc88478640d7e76992438d90d3d6a68a90 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 24 Jul 2021 12:53:39 +0200 Subject: [PATCH 049/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 1273e432..181c4139 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,7 @@ next: * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) + * Build: Improve Makefile compatibility (Evilham) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 4156fea9001aa267c8b173247cada2f919511c1b Mon Sep 17 00:00:00 2001 From: Joachim Desroches Date: Wed, 28 Jul 2021 12:56:39 +0200 Subject: [PATCH 050/128] [filesystem] Add ubuntu as supported distribution. --- cdist/conf/type/__filesystem/explorer/lsblk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index 9be3c575..d376c09f 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|redhat|suse|gentoo) + alpine|centos|fedora|gentoo|redhat|suse|ubuntu) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 From 542674dae81ab03a4de2c4ad0b2eaf264ee2c442 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 30 Jul 2021 10:30:33 +0200 Subject: [PATCH 051/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 181c4139..9fc10b20 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) + * Type __filesystem: Support ubuntu (Joachim Desroches) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 53334fb4eb311550af7f5b73f279a2e86fa1c504 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 19:50:10 +0200 Subject: [PATCH 052/128] [explorer/os_version] Fix for FreeBSD < 10.0 (again) --- cdist/conf/explorer/os_version | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..cc976608 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -89,7 +89,14 @@ in freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - freebsd-version + if command -v freebsd-version >/dev/null 2>&1 + then + # get userland version + freebsd-version -u + else + # fallback to kernel release for FreeBSD < 10.0 + uname -r + fi ;; *bsd|solaris) uname -r From e108cbc205cb5e7ac0d2e07b82cef4c83eaa285f Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 13:20:43 +0200 Subject: [PATCH 053/128] [explorer/os_version] Ubuntu: fall back to os-release/lsb-release files --- cdist/conf/explorer/os_version | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 6c94915c..e70fe7f4 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -21,6 +21,17 @@ # All os variables are lower case # +rc_getvar() { + awk -F= -v varname="$2" ' + function unquote(s) { + if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) + return substr(s, 2, length(s) - 2) + else + return s + } + $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" +} + case $("${__explorer:?}/os") in amazon) @@ -114,7 +125,20 @@ in fi ;; ubuntu) - lsb_release -sr + if command -v lsb_release >/dev/null 2>&1 + then + lsb_release -sr + elif test -r /usr/lib/os-release + then + # fallback to /usr/lib/os-release if lsb_release is not present (like + # on minimized Ubuntu installations) + rc_getvar /usr/lib/os-release VERSION_ID + elif test -r /etc/lsb-release + then + # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old + # versions without /usr/lib/os-release. + rc_getvar /etc/lsb-release DISTRIB_RELEASE + fi ;; alpine) cat /etc/alpine-release From 83fe6e9f5b2537db73d5c6a142b6d24eef75ac58 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Tue, 3 Aug 2021 19:26:55 +0200 Subject: [PATCH 054/128] [explorer/memory] Fix conversion of large numbers (>= 2GiB) At least mawk uses scientific notation when using print for numbers >=2^31 (INT_MAX of a signed 32-bit int). `printf "%.f\n"` works around this. --- cdist/conf/explorer/memory | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 63aba9c6..c6d113cf 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,19 +27,18 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { print $1 * 1000 } - $2 == "MB" { print $1 * 1000 * 1000 } - $2 == "GB" { print $1 * 1000 * 1000 * 1000 } - $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } - $2 == "kiB" { print $1 * 1024 } - $2 == "MiB" { print $1 * 1024 * 1024 } - $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } - $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' + $2 == "kB" { printf "%.f\n", ($1 * 1000) } + $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } + $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } + $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } + $2 == "kiB" { printf "%.f\n", ($1 * 1024) } + $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } + $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } + $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' } bytes2kib() { - set -- "$(cat)" - test "$1" -gt 0 && echo $(($1 / 1024)) + awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' } From a7d6481a7ddc7cb72b1a55bfca7fdfed20514a62 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:23:50 +0200 Subject: [PATCH 055/128] [type/__update_alternatives] Secure cdist-defined environment variables with :? --- .../__update_alternatives/explorer/alternatives | 2 +- cdist/conf/type/__update_alternatives/explorer/link | 6 +++--- .../type/__update_alternatives/explorer/path_is | 4 ++-- .../explorer/path_should_state | 2 +- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++------- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index 34aaca56..ecc62f4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "$__object_id" 2>/dev/null \ +update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index 6519e7c2..c6fd1c98 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/$__object_id" ] + if [ ! -f "$altdir/${__object_id:?}" ] then continue fi - link="$( awk 'NR==2' "$altdir/$__object_id" )" + link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" if [ -n "$link" ] then @@ -33,7 +33,7 @@ done if [ -z "$link" ] then - echo "unable to get link for $__object_id" >&2 + echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index fc304d5d..a24bd40e 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,11 +1,11 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ +path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" if [ -z "$path_is" ] then - echo "unable to get current path for $__object_id" >&2 + echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index 59e015c5..b74a7ee8 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "$__object/parameter/path" )" ] +if [ -f "$( cat "${__object:?}/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e393cdef..13666805 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,26 +18,25 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "$__object/explorer/path_is" )" +path_is="$( cat "${__object:?}/explorer/path_is" )" -path_should="$( cat "$__object/parameter/path" )" +path_should="$( cat "${__object:?}/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] +if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ + && [ -z "${__cdist_dry_run+dry run}" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name="$__object_id" +name=${__object_id:?} -alternatives="$( cat "$__object/explorer/alternatives" )" - -if ! echo "$alternatives" | grep -Fxq "$path_should" +if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then if [ ! -f "$__object/parameter/install" ] then From 0b3b47396f2aafa377e3d5d9a13f51ace2303d41 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Mon, 2 Aug 2021 21:25:08 +0200 Subject: [PATCH 056/128] [type/__update_alternatives] dry-run fixes --- cdist/conf/type/__update_alternatives/explorer/link | 5 ++++- .../type/__update_alternatives/explorer/path_is | 5 ++++- .../conf/type/__update_alternatives/gencode-remote | 13 ++++++++----- 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index c6fd1c98..d1087c75 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -31,8 +31,11 @@ do fi done -if [ -z "$link" ] +if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the link + # might be managed by another cdist object (which wasn't executed, + # because dry run…). echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index a24bd40e..9208df7b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -3,8 +3,11 @@ path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] +if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then + # NOTE: ignore error for dry-runs because a package providing the + # alternative might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index 13666805..e91ea78f 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -38,16 +38,19 @@ name=${__object_id:?} if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then - if [ ! -f "$__object/parameter/install" ] + if [ -f "${__object:?}/parameter/install" ] then + link="$( cat "${__object:?}/explorer/link" )" + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" + elif [ -z "${__cdist_dry_run+dry run}" ] + then + # NOTE: ignore error for dry-runs because a package providing the link + # to be installed might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi - - link="$( cat "$__object/explorer/link" )" - - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" From bbcc81a9841f2619e1b9e13b25a941337489a681 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:44:04 +0200 Subject: [PATCH 057/128] [type/__update_alternatives] Fix for non-English locales Since update-alternatives(1) is localized, screen scraping its output breaks if the locale is set to non-English. --- cdist/conf/type/__update_alternatives/explorer/alternatives | 4 ++-- cdist/conf/type/__update_alternatives/explorer/path_is | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index ecc62f4b..bb1619a9 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ +| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 9208df7b..5cf4fa4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,7 +1,8 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "${__object_id:?}" 2>/dev/null \ - | awk '/link currently points to/ {print $5}' )" +path_is=$( + LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ + | awk '/link currently points to/ { print $5 }') if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then From 2a0c073d4021206c7459015cefaba218004235ce Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 4 Aug 2021 21:54:17 +0200 Subject: [PATCH 058/128] [explorer/os_version] Fix for legacy Mac OS X versions --- cdist/conf/explorer/os_version | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..96eca1ee 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -68,7 +68,8 @@ case "$("$__explorer/os")" in cat /etc/gentoo-release ;; macosx) - sw_vers -productVersion + # NOTE: Legacy versions (< 10.3) do not support options + sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. From 3ae5a606ca7182ec7fe13134670400506e198ec1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 5 Aug 2021 10:27:51 +0200 Subject: [PATCH 059/128] ++changelog --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 9fc10b20..f9409d7e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,10 @@ next: * Core: Fix logging bug (Dennis Camera) * Build: Improve Makefile compatibility (Evilham) * Type __filesystem: Support ubuntu (Joachim Desroches) + * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) + * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) + * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) + * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 21 Jul 2021 13:22:34 +0200 Subject: [PATCH 060/128] [explorer/machine_type] Reimplement --- cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++--- 1 file changed, 904 insertions(+), 82 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1c84f4d7..29f98849 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,8 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Daniel Heule (hda at sfs.biz) -# 2014 Thomas Oettli (otho at sfs.biz) -# 2020 Evilham (contact at evilham.com) +# 2021 Dennis Camera (cdist at dtnr.ch) # # This file is part of cdist. # @@ -19,91 +17,915 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to determine what type of machine the target to be +# configured is (container, virtual machine, bare-metal). +# +# It will print one line for each layer it can detect. +# The format of all lines is: TYPE[ VERB VENDOR] +# +# VERB does not have a special meaning, it is just for better readability. +# +# e.g. +# container on lxc +# virtual by kvm-spapr +# +# The third word of each line can be composed of different parts concatenated with a `-' +# (minus) character, with each component being a specification of the previous, +# e.g.: +# - lxc-libvirt (LXC container, managed by libvirt) +# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) +# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) +# +# If this explorer cannot determine any information it will print nothing. +# -os=$("$__explorer/os") +# Add /sbin and /usr/sbin to the path so we can find system +# binaries like dmidecode. +PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' +PATH="/sbin:/usr/sbin:${PATH}" +export PATH -vendor_string_to_machine_type() { - for vendor in vmware bochs kvm qemu virtualbox bhyve; do - if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then - vendor="kvm" - fi - echo "virtual_by_${vendor}" - exit - fi - done +arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') +uname_s=$(uname -s) + + +is_command() { command -v "$1" >/dev/null 2>&1; } + +is_oneof() ( + x=$1; shift + for y + do + test "${x}" = "${y}" || continue + return 0 + done + return 1 +) + +tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } + +# shellcheck disable=SC2086 +glob_exists() { set -- $1; test -e "$1"; } + +get_dmi_field() { + if is_oneof "${uname_s}" NetBSD + then + case $1 + in + (system-manufacturer) _mib=machdep.dmi.system-vendor ;; + (system-product-name) _mib=machdep.dmi.system-product ;; + (system-version|system-uuid) _mib=machdep.dmi.$1 ;; + (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; + (biod-release-date) _mib=machdep.dmi.bios-date ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + if is_command dmidecode + then + dmidecode -s "$1" + elif test -d "${dmi_sysfs-}" + then + case $1 + in + (system-manufacturer) _filename=sys_vendor ;; + (system-product-name) _filename=product_name ;; + (*) _filename=$(echo "$1" | tr - _) ;; + esac + if test -r "${dmi_sysfs-}/${_filename}" + then + cat "${dmi_sysfs}/${_filename}" + fi + unset _filename + elif test "${uname_s}" = OpenBSD + then + # NOTE: something similar to system-manufacutrer and system-product-name + # is available on OpenBSD in sysctl + case $1 + in + (system-manufacturer) _mib=hw.vendor ;; + (system-product-name) _mib=hw.product ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + return 1 } -case "$os" in - "freebsd") - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" - if [ -n "${vm_guest}" ]; then - if [ "${vm_guest}" = "none" ]; then - echo "physical" - exit - fi - echo "virtual_by_${vm_guest}" - exit - fi - ;; +has_cpuinfo() { test -e /proc/cpuinfo; } - "openbsd") - # OpenBSD can also use the sysctl's: hw.vendor or hw.product. - # Note we can be reasonably sure about a machine being virtualised - # as long as we can identify the virtualisation technology. - # But not so much about it being physical... - # Patches are welcome / reach out if you have better ideas. - for sysctl in hw.vendor hw.product; do - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" - done - ;; +get_sysctl() { + is_command sysctl && sysctl -n "$1" 2>/dev/null +} - *) - # Defaulting to linux for compatibility with previous cdist behaviour - if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit - fi +# Check for container - if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit - fi +has_ct_pid_1() { + test -r /run/systemd/container -o -r /proc/1/environ +} - if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm, bhyve - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - for vendor_file in /sys/class/dmi/id/product_name \ - /sys/class/dmi/id/sys_vendor \ - /sys/class/dmi/id/chasis_vendor; do - if [ -r ${vendor_file} ]; then - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(cat "${vendor_file}")" - fi - done - fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi - fi - ;; -esac +translate_container_name() { + case $1 + in + ('lxc') + echo lxc ;; + ('lxc-libvirt') + echo lxc-libvirt ;; + ('podman') + echo podman ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + return 1 ;; + esac + return 0 +} -echo "unknown" +check_ct_pid_1() { + if test -r /run/systemd/container + then + translate_container_name "$(head -n1 /run/systemd/container)" \ + && return 0 + fi + + if test -r /proc/1/environ + then + translate_container_name "$( + LC_ALL=C tr '\000' '\n' /dev/null + then + # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + echo wsl + elif test -d /var/.cagefs + then + # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs + # CageFS is not "really" a container, but it isn't a chroot either. + echo cagefs + elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status + then + # Linux-VServer + if grep -q -x -F 'VxID: 0' /proc/self/status + then + # host + return 1 + else + # guest + echo linux_vserver + fi + else + return 1 + fi +} + +check_ct_os_specific() ( + if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 + then + # FreeBSD jail + echo jail + return 0 + fi + + if is_command zonename && test "$(zonename)" != global + then + # Solaris zone + echo zone + return 0 + fi + + return 1 +) + + +# Check for hypervisor + +guess_hypervisor_from_cpu_model() { + case $1 + in + (*\ KVM\ *) + echo kvm ;; + (*\ QEMU\ *|QEMU\ *) + echo qemu ;; + (*) + return 1 ;; + esac +} + +has_vm_cpuinfo() { has_cpuinfo; } + +check_vm_cpuinfo() { + if grep -q -F 'User Mode Linux' /proc/cpuinfo \ + || grep -q -F 'UML' /proc/cpuinfo + then + # User Mode Linux + echo uml + elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo + then + # IBM PowerVM Lx86 (Linux/x86 emulator) + echo powervm_lx86 + elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo + then + # IBM SystemZ (S/390) + if test -f /proc/sysinfo + then + if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo + then + echo kvm-s390 + return 0 + elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo + then + echo zvm + return 0 + elif grep -q -e '^LPAR ' /proc/sysinfo + then + echo zvm-lpar + return 0 + fi + fi + return 1 + else + if grep -q -e '^model name.*:' /proc/cpuinfo + then + sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ + | while read -r _cpu_model + do + guess_hypervisor_from_cpu_model "${_cpu_model}" + done \ + | sort \ + | uniq -c \ + | awk ' + { if ($1 > most_c) { most_c = $1; most_s = $2 } } + END { + if (most_s) print most_s + exit !most_s + }' \ + && return 0 + fi + return 1 + fi +} + +check_vm_arch_specific() { + case ${arch} + in + (ppc64|ppc64le) + # Check PPC64 LPAR, KVM + + # example /proc/cpuinfo line indicating 'not baremetal' + # platform : pSeries + # + # example /proc/ppc64/lparcfg systemtype line + # system_type=IBM pSeries (emulated by qemu) + + if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo + then + if test -e /proc/ppc64/lparcfg + then + # Assume LPAR, now detect shared or dedicated + if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg + then + echo powervm-shared + return 0 + else + echo powervm-dedicated + return 0 + fi + fi + fi + ;; + (sparc*) + # Check for SPARC LDoms + + if test -e /dev/mdesc + then + if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp + then + # control LDom + return 1 + else + # guest LDom + echo ldom-sparc + fi + + # MDPROP=/usr/lib/ldoms/mdprop.py + # if test -x "${MDPROP}" + # then + # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" + # then + # echo ldoms-root + # echo ldoms-io + # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" + # then + # echo ldoms-io + # fi + # fi + return 0 + fi + ;; + (i?86|x86*|amd64|i86pc) + # Check CPUID + # + # Many fullvirt hypervisors give an indication through CPUID. Use + # the virt-what helper program to get this information if available. + + for CPUID_HELPER in \ + $(command -v virt-what-cpuid-helper 2>/dev/null) \ + /usr/lib/x86_64-*/virt-what-cpuid-helper \ + /usr/lib/i?86-*/virt-what-cpuid-helper \ + /usr/lib/virt-what/virt-what-cpuid-helper + do + if test -x "${CPUID_HELPER:?}"; then break; fi + done + + if test -x "${CPUID_HELPER-}" + then + case $(command "${CPUID_HELPER}") + in + ('bhyve bhyve ') + echo bhyve + ;; + ('LKVMLKVMLKVM') + echo lkvm + ;; + ('KVMKVMKVM') + echo kvm + ;; + ('TCGTCGTCGTCG') + echo qemu-tcg + ;; + ('Microsoft Hv') + # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx + echo hyperv + ;; + ('OpenBSDVMM58') + # OpenBSD/VMM + echo openbsd_vmm + ;; + ('VMwareVMware') + # check added by Chetan Loke. + echo vmware + ;; + ('XenVMMXenVMM') + if has dmi + then + # https://access.redhat.com/solutions/222903 + echo xen-hvm + else + echo xen-paravirt + fi + ;; + (*) + return 1 ;; + esac + return 0 + fi + + unset CPUID_HELPER + + # VMM CPUID flag denotes that this system is running under a VMM + if is_oneof "${uname_s}" Darwin + then + get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ + && return 0 + fi + if has_cpuinfo \ + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + then + return 0 + fi + ;; + (ia64) + if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend + then + # PV-on-HVM drivers installed in a Xen guest + echo xen-hvm + return 0 + fi + ;; + esac + return 1 +} + +has_vm_dmi() { + # Check for various products in SMBIOS/DMI. + # Note that DMI doesn't exist on all architectures (only x86 and some ARM). + # On other architectures the $dmi variable will be empty. + + if test -d /sys/class/dmi/id/ + then + dmi_sysfs=/sys/class/dmi/id + elif test -d /sys/devices/virtual/dmi/id/ + then + dmi_sysfs=/sys/devices/virtual/dmi/id + fi + + # shellcheck disable=SC2015 + { + is_command dmidecode \ + && ( + # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line + dmi_out=$(dmidecode 2>&1) \ + && ! printf '%s\n' "${dmi_out}" \ + | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' + ) \ + || test -d "${dmi_sysfs}" + } +} + +check_vm_dmi() { + case $(get_dmi_field system-product-name) + in + (*.metal) + if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' + then + # AWS EC2 bare metal -> no virtualisation + return 1 + fi + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('Google Compute Engine') + echo gce + return 0 + ;; + ('RHEV Hypervisor') + # Red Hat Enterprise Virtualization + echo rhev + return 0 + ;; + ('KVM'|'Bochs'|'KVM Virtual Machine') + echo kvm + return 0 + ;; + ('Parallels Virtual Platform') + echo parallels + return 0 + ;; + ('VirtualBox') + echo virtualbox + return 0 + ;; + ('VMware Virtual Platform') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field system-manufacturer) + in + ('Alibaba'*) + case $(get_dmi_field system-product-name) + in + ('Alibaba Cloud ECS') + echo alibaba-ecs + ;; + (*) + echo alibaba + ;; + esac + return 0 + ;; + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('innotek GmbH'|'Oracle Corporation') + echo virtualbox + return 0 + ;; + ('Joyent') + if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' + then + # SmartOS KVM + echo kvm-smartdc_hvm + return 0 + fi + ;; + ('Microsoft Corporation'*) + if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' + then + if test -e /proc/irq/7/hyperv \ + || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null + then + echo hyperv + return 0 + fi + + case $(get_dmi_field system-version) + in + (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) + echo virtualpc + return 0 + ;; + (*) + echo hyperv + return 0 + ;; + esac + fi + ;; + ('Nutanix') + # Nutanix AHV. Similar to KVM. + if test "$(get_dmi_field system-product-name)" = 'AHV' + then + echo nutanix_ahv + return 0 + fi + ;; + ('oVirt') + echo ovirt + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('QEMU') + echo qemu + return 0 + ;; + ('VMware, Inc.') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field bios-vendor) + in + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('innotek GmbH') + echo virtualbox + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('Xen') + if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' + then + # AWS on Xen + echo aws-xen + return 0 + fi + ;; + esac + + return 1 +} + +check_vm_hyp_specific() { + if is_command vmware-checkvm && vmware-checkvm >/dev/null + then + # vmware-checkvm is provided by VMware's open-vm-tools + echo vmware + return 0 + elif test -d /proc/xen + then + test -r /proc/xen/capabilities && + if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null + then + # Xen dom0 + return 1 + else + # Xen domU + echo xen + return 0 + fi + fi + return 1 +} + +has_vm_dt() { + # OpenFirmware/Das U-Boot device-tree + test -d /proc/device-tree +} + +check_vm_dt() { + case ${arch} + in + (arm|aarch64) + if test -r /proc/device-tree/hypervisor/compatible + then + if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible + then + echo xen + return 0 + elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible + then + # e.g. VMware ESXi on ARM + echo vmware + return 0 + fi + fi + if glob_exists /proc/device-tree/fw-cfg@*/compatible + then + # qemu,fw-cfg-mmio + sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 + return 0 + fi + if grep -q -F 'dummy-virt' /proc/device-tree/compatible + then + echo lkvm + return 0 + fi + ;; + (ppc64*) + if test -d /proc/device-tree/hypervisor \ + && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible + then + # We are running as a spapr KVM guest on ppc64 + echo kvm-spapr + return 0 + fi + if test -r /proc/device-tree/ibm,partition-name \ + && test -r /proc/device-tree/hmc-managed\? \ + && test -r /proc/device-tree/chosen/qemu,graphic-width + then + echo powervm + fi + ;; + esac + return 1 +} + +has_vm_sys_hypervisor() { + test -d /sys/hypervisor/ +} + +check_vm_sys_hypervisor() { + test -r /sys/hypervisor/type && + case $(head -n1 /sys/hypervisor/type) + in + (xen) + # Ordinary kernel with pv_ops. There does not seem to be + # enough information at present to tell whether this is dom0 + # or domU. + echo xen + return 0 + ;; + esac + return 1 +} + +check_vm_os_specific() { + _hyp_generic=false + + case ${uname_s} + in + (Darwin) + if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ + && test "${hv_vmm_present}" -ne 0 + then + _hyp_generic=true + fi + ;; + (FreeBSD) + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest=$(get_sysctl kern.vm_guest | tolower) && + case ${vm_guest} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${vm_guest} + in + (hv) echo hyperv ;; + (vbox) echo virtualbox ;; + (*) echo "${vm_guest}" ;; + esac + return 0 + ;; + esac + ;; + (NetBSD) + machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && + case ${machdep_hv} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${machdep_hv} + in + (hyper-v) echo hyperv ;; + (xenhvm*) echo xen-hvm ;; + (xenpv*) echo xen-pv ;; + (xen*) echo xen ;; + (*) echo "${machdep_hv}" ;; + esac + return 0 + ;; + esac + ;; + (OpenBSD) + if is_command hostctl && glob_exists /dev/pvbus[0-9]* + then + for _pvbus in /dev/pvbus[0-9]* + do + _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue + case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') + in + (KVM) echo kvm ;; + (Hyper-V) echo hyperv ;; + (VMware) echo vmware ;; + (Xen) echo xen ;; + (bhyve) echo bhyve ;; + (OpenBSD) echo openbsd_vmm ;; + esac + return 0 + done + fi + ;; + (SunOS) + diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') + # NOTE: Don't use -e or -F in Solaris grep + if printf '%s\n' "${diag_conf}" | grep -q -i QEMU + then + echo qemu + return 0 + elif printf '%s\n' "${diag_conf}" | grep -q -i VMware + then + echo vmware + return 0 + fi + ;; + (Linux) + if is_command dmesg + then + while read -r line + do + case ${line} + in + ('Booting paravirtualized kernel on ') + case $(expr "${line}" : '.* kernel on \(.*\)') + in + ('Xen') + echo xen-pv; return 0 ;; + ('bare hardware') + return 1 ;; + esac + ;; + ('Hypervisor detected') + case $(expr "${line}" : '.*: *\(.*\)') + in + ('ACRN') + echo acrn ;; + ('Jailhouse') + echo jailhouse ;; + ('KVM') + echo kvm ;; + ('Microsoft Hyper-V') + echo hyperv ;; + ('VMware') + echo vmware ;; + ('Xen HVM') + echo xen-hvm ;; + ('Xen PV') + echo xen-pv ;; + esac + return 0 + ;; + (lpar:*' under hypervisor') + return 0 ;; + esac + done <<-EOF + $(dmesg 2>/dev/null | awk ' + /Booting paravirtualized kernel on / + /Hypervisor detected: / + /lpar: .* under hypervisor/ + ') + EOF + fi + esac + + # Try to guess hypervisor based on CPU model (sysctl hw.model if available) + if cpu_model=$(get_sysctl hw.model) + then + guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 + fi + + if ${_hyp_generic} + then + # cannot say which hypervisor, but one was detected + return 0 + else + return 1 + fi +} + +run_stage() { + if type "has_$1_$2" >/dev/null 2>&1 + then + "has_$1_$2" + else + true + fi \ + && "check_$1_$2" +} + + +# Execute container stages + +for stage in \ + pid_1 cgroup files os_specific +do + ctengine=$(run_stage ct ${stage}) || continue + is_contained=true + if test -n "${ctengine}" + then + echo container on "${ctengine}" + break + fi +done +if ${is_contained:-false} && test -z "${ctengine}" +then + # none of the stages could determine the specific container engine, but + # we are running in some container. + echo container +fi + + +# Execute virtual machine / hypervisor stages + +for stage in \ + os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific +do + hypervisor=$(run_stage vm ${stage}) || continue + is_virtual=true + if test -n "${hypervisor}" + then + echo virtual by "${hypervisor}" + break + fi +done +if ${is_virtual:-false} && test -z "${hypervisor}" +then + # none of the stages could determine the specific hypervisor, but + # we are virtual. + echo virtual +fi From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:29:41 +0200 Subject: [PATCH 061/128] [explorer/machine_type] Print top most machine layer as first line (fallback to physical) --- cdist/conf/explorer/machine_type | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 29f98849..90c441da 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -26,17 +26,19 @@ # VERB does not have a special meaning, it is just for better readability. # # e.g. +# container # container on lxc # virtual by kvm-spapr # -# The third word of each line can be composed of different parts concatenated with a `-' -# (minus) character, with each component being a specification of the previous, -# e.g.: +# The third word of each line (except the first) can be composed of different +# parts concatenated with a `-' (minus) character, with each component being +# a specification of the previous, e.g.: # - lxc-libvirt (LXC container, managed by libvirt) # - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) # - xen-hvm / xen-pv (Xen HVM vs para-virtualization) # -# If this explorer cannot determine any information it will print nothing. +# If this explorer cannot collect enough information about virtualization it +# will fall back to 'physical'. # # Add /sbin and /usr/sbin to the path so we can find system @@ -121,6 +123,10 @@ get_sysctl() { is_command sysctl && sysctl -n "$1" 2>/dev/null } +detected_layer() { + test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" +} + # Check for container @@ -895,6 +901,7 @@ for stage in \ pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue + detected_layer 'container' is_contained=true if test -n "${ctengine}" then @@ -916,6 +923,7 @@ for stage in \ os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue + detected_layer 'virtual machine' is_virtual=true if test -n "${hypervisor}" then @@ -929,3 +937,8 @@ then # we are virtual. echo virtual fi + + +# Fallback + +detected_layer physical From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:57:24 +0200 Subject: [PATCH 062/128] [explorer/machine_type] Remove CPUID check it's a lot of code and depends on a binary helper unlikely to be installed. --- cdist/conf/explorer/machine_type | 59 -------------------------------- 1 file changed, 59 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 90c441da..7ce035e3 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -379,65 +379,6 @@ check_vm_arch_specific() { fi ;; (i?86|x86*|amd64|i86pc) - # Check CPUID - # - # Many fullvirt hypervisors give an indication through CPUID. Use - # the virt-what helper program to get this information if available. - - for CPUID_HELPER in \ - $(command -v virt-what-cpuid-helper 2>/dev/null) \ - /usr/lib/x86_64-*/virt-what-cpuid-helper \ - /usr/lib/i?86-*/virt-what-cpuid-helper \ - /usr/lib/virt-what/virt-what-cpuid-helper - do - if test -x "${CPUID_HELPER:?}"; then break; fi - done - - if test -x "${CPUID_HELPER-}" - then - case $(command "${CPUID_HELPER}") - in - ('bhyve bhyve ') - echo bhyve - ;; - ('LKVMLKVMLKVM') - echo lkvm - ;; - ('KVMKVMKVM') - echo kvm - ;; - ('TCGTCGTCGTCG') - echo qemu-tcg - ;; - ('Microsoft Hv') - # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx - echo hyperv - ;; - ('OpenBSDVMM58') - # OpenBSD/VMM - echo openbsd_vmm - ;; - ('VMwareVMware') - # check added by Chetan Loke. - echo vmware - ;; - ('XenVMMXenVMM') - if has dmi - then - # https://access.redhat.com/solutions/222903 - echo xen-hvm - else - echo xen-paravirt - fi - ;; - (*) - return 1 ;; - esac - return 0 - fi - - unset CPUID_HELPER - # VMM CPUID flag denotes that this system is running under a VMM if is_oneof "${uname_s}" Darwin then From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 21:29:24 +0200 Subject: [PATCH 063/128] [explorer/machine_type] Use systemd-detect-virt (if available) to detect containers and VMs --- cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 7ce035e3..1a38fda0 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -130,6 +130,25 @@ detected_layer() { # Check for container +has_ct_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -c' +} + +check_ct_systemd() ( + _ctengine=$(systemd-detect-virt -c 2>/dev/null) && + case ${_ctengine} + in + (''|'none') + return 1 ;; + ('container-other') + return 0 ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + echo "${_ctengine}" ;; + esac +) + has_ct_pid_1() { test -r /run/systemd/container -o -r /proc/1/environ } @@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() { esac } +has_vm_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' +} + +check_vm_systemd() ( + _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && + case ${_hypervisor} + in + (''|'none') + return 1 ;; + ('amazon') + echo aws ;; + ('bochs') + echo kvm ;; + ('microsoft') + # assumption + echo hyperv ;; + ('oracle') + echo virtualbox ;; + ('vm-other') + return 0 ;; + (*) + echo "${_hypervisor}" ;; + esac +) + has_vm_cpuinfo() { has_cpuinfo; } check_vm_cpuinfo() { @@ -839,7 +884,7 @@ run_stage() { # Execute container stages for stage in \ - pid_1 cgroup files os_specific + systemd pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue detected_layer 'container' @@ -861,7 +906,7 @@ fi # Execute virtual machine / hypervisor stages for stage in \ - os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific + systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue detected_layer 'virtual machine' From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 22:01:28 +0200 Subject: [PATCH 064/128] [explorer/machine_type] Implement chroot detection --- cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1a38fda0..10c914ba 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -53,6 +53,21 @@ uname_s=$(uname -s) is_command() { command -v "$1" >/dev/null 2>&1; } +files_same() { + # shellcheck disable=SC2012 + LC_ALL=C df -P "$1" "$2" 2>/dev/null | { + read -r _ # skip header line + read -r fs1 _ _ _ _ mp1 + read -r fs2 _ _ _ _ mp2 + test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 + } && + ls -1Ldi "$1" "$2" 2>/dev/null | { + read -r ino1 _ + read -r ino2 _ + test "${ino1}" = "${ino2}" || return 1 + } +} + is_oneof() ( x=$1; shift for y @@ -128,6 +143,32 @@ detected_layer() { } +# Check for chroot + +has_chroot_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' +} + +check_chroot_systemd() { + systemd-detect-virt -r +} + +has_chroot_debian_ischroot() { + is_command ischroot +} + +check_chroot_debian_ischroot() { + ischroot --default-false +} + +has_chroot_procfs() { + test -d /proc/ +} + +check_chroot_procfs() { + test -e /proc/1/root && ! files_same /proc/1/root / +} + # Check for container has_ct_systemd() { @@ -881,6 +922,18 @@ run_stage() { } +# Execute chroot stages + +for stage in \ + procfs debian_ischroot systemd +do + run_stage chroot ${stage} || continue + detected_layer 'chroot' + echo chroot + break +done + + # Execute container stages for stage in \ From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 16:40:20 +0200 Subject: [PATCH 065/128] [explorer/machine_type] Try to detect chroot path --- cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 10c914ba..fa68ec4d 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -166,7 +166,28 @@ has_chroot_procfs() { } check_chroot_procfs() { - test -e /proc/1/root && ! files_same /proc/1/root / + if test -e /proc/1/root && ! files_same /proc/1/root / + then + # try to determine where the chroot has been mounted + ( + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # Get mount point + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + fi + ) + return 0 + fi + return 1 } # Check for container @@ -927,11 +948,20 @@ run_stage() { for stage in \ procfs debian_ischroot systemd do - run_stage chroot ${stage} || continue + chrootpnt=$(run_stage chroot ${stage}) || continue + is_chrooted=true detected_layer 'chroot' - echo chroot - break + if test -n "${chrootpnt}" + then + echo chroot at "${chrootpnt}" + break + fi done +if ${is_chrooted:-false} && test -z "${chrootpnt}" +then + # could determine chroot, but not its mount point + echo chroot +fi # Execute container stages From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 23:09:02 +0200 Subject: [PATCH 066/128] [explorer/machine_type] Implement chroot detection using /proc/.../mountinfo --- cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index fa68ec4d..00646c75 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -165,30 +165,52 @@ has_chroot_procfs() { test -d /proc/ } -check_chroot_procfs() { +check_chroot_procfs() ( + is_chroot=false # default if test -e /proc/1/root && ! files_same /proc/1/root / then - # try to determine where the chroot has been mounted - ( - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # Get mount point - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - fi - ) - return 0 + is_chroot=true fi - return 1 -} + if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo + then + has_mountinfo=true + cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true + fi + + if ${is_chroot} + then + # try to determine where the chroot has been mounted + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # escape chroot and find mount point by inode + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + elif ${has_mountinfo} + then + while read -r mntid _ _ _ cmntpnt _ + do + read -r _ _ _ _ hmntpnt _ <<-EOF + $(grep -e "^$((mntid)) " /proc/1/mountinfo) + EOF + printf '%s\n' "${hmntpnt%${cmntpnt}}" + done Date: Mon, 23 Aug 2021 09:57:20 +0300 Subject: [PATCH 067/128] [explorer/os_version] add new debian code names: bookworm and trixie --- cdist/conf/explorer/os_version | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 7bc6dd6b..aea3c43f 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -54,6 +54,8 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in + trixie) echo 12.99 ;; + bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; From e1e134899811e51e4694e89e421790e41cb894bf Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 23 Aug 2021 10:44:48 +0300 Subject: [PATCH 068/128] [explorer/os_version] use 99.99 as fallback for unknown code names in */sid --- cdist/conf/explorer/os_version | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index aea3c43f..bbc9e4f0 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -63,7 +63,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) exit 1 + *) echo 99.99 ;; esac ;; *) From 46ed48d546af6ff663b131c803a9027b19798ba1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 08:09:47 +0200 Subject: [PATCH 069/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f9409d7e..8507c663 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) + * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:32:44 +0200 Subject: [PATCH 070/128] Update shellcheck disable --- cdist/conf/type/__apt_backports/manifest | 1 + cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 + cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 3 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index bc47d8de..6fcd9212 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,6 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 3f88f6bc..ca9aa45a 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,6 +37,7 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 + # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index cec25746..9694a64b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,6 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1090 +# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:25:49 +0200 Subject: [PATCH 071/128] Release 6.9.8 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8507c663..dcdc4b3d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:47:50 +0200 Subject: [PATCH 072/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index dcdc4b3d..693d028f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Explorer machine_type: Rewrite (Dennis Camera) + 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 21:54:45 +0300 Subject: [PATCH 073/128] allow empty file --- cdist/conf/type/__sed/explorer/file | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index a40f71b3..5f0f4edd 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,13 +9,7 @@ fi if [ -f "$file" ] then - if [ -s "$file" ] - then - cat "$file" - else - echo "$file is empty" >&2 - exit 1 - fi + cat "$file" else echo "$file do not exist" >&2 exit 1 From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:24:26 +0300 Subject: [PATCH 074/128] use $__object/tempfile in target instead of mktemp, add comments --- cdist/conf/type/__sed/gencode-remote | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 836506f0..df929b42 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -35,8 +35,11 @@ then exit 0 fi +# we can't use -i, because it's not posix, so we fly with tempfile and cp +# and we use cp because we want to preserve destination file's attributes + # shellcheck disable=SC2016 -echo 'tmp="$( mktemp )"' +echo 'tmp="$__object/tempfile"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:27:42 +0300 Subject: [PATCH 075/128] use -e --- cdist/conf/type/__sed/explorer/file | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 5f0f4edd..9262b613 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -7,10 +7,10 @@ else file="/$__object_id" fi -if [ -f "$file" ] +if [ ! -e "$file" ] then - cat "$file" -else echo "$file do not exist" >&2 exit 1 fi + +cat "$file" From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:38:55 +0300 Subject: [PATCH 076/128] use -E for better compat (not really sure if it is posix at all) --- cdist/conf/type/__sed/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index df929b42..625448e6 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -24,7 +24,7 @@ sed_cmd='sed' if [ -f "$__object/parameter/regexp-extended" ] then - sed_cmd="$sed_cmd --regexp-extended" + sed_cmd="$sed_cmd -E" fi if echo "$script" \ From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:40:06 +0300 Subject: [PATCH 077/128] remove reading script from file --- cdist/conf/type/__sed/gencode-remote | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 625448e6..be7690e1 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )" if [ "$script" = '-' ] then script="$( cat "$__object/stdin" )" -elif - [ -f "$script" ] -then - script="$( cat "$script" )" fi file_from_target="$__object/explorer/file" From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:45:36 +0300 Subject: [PATCH 078/128] update man --- cdist/conf/type/__sed/man.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index 4a728184..86789363 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed`` DESCRIPTION ----------- -TODO +Transform text files with ``sed``. REQUIRED MULTIPLE PARAMETERS ---------------------------- script - TODO + ``sed`` script. + If ``-`` then the script is read from ``stdin``. OPTIONAL PARAMETERS ------------------- file - TODO + Path to the file. Defaults to ``$__object_id``. onchange - TODO + Execute this command if ``sed`` changes file. BOOLEAN PARAMETERS ------------------ regexp-extended - TODO + Use extended regular expressions in the script. + Might not be supported with every ``sed`` version. EXAMPLES -------- + .. code-block:: sh - true + __sed /tmp/foobar --script 's/foo/bar/' + + echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - AUTHORS From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Sep 2021 09:22:27 +0300 Subject: [PATCH 079/128] grammar --- cdist/conf/type/__sed/explorer/file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 9262b613..ec3d0fe8 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,7 +9,7 @@ fi if [ ! -e "$file" ] then - echo "$file do not exist" >&2 + echo "$file does not exist" >&2 exit 1 fi From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:04:12 +0200 Subject: [PATCH 080/128] __package_update_index: fix complain about suite change 2 of 4th fix for ticket #861 --- .../type/__package_update_index/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 803468b5..a10c16d3 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,7 +41,19 @@ fi case "$type" in yum) ;; apt) - echo "apt-get --quiet update" + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + + echo "apt-get --quiet $apt_opts update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:12:20 +0200 Subject: [PATCH 081/128] __apt_source: fix complain about suite change 3 of 4th fix for ticket #861 --- cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 1e8592c6..973b0f6c 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,7 +22,21 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + +# run 'apt-get update' only if something changed with our sources.list file +# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get update || apt-get update\n' + printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" fi From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 14:49:23 +0200 Subject: [PATCH 082/128] __apt_update_index: fix complain about suite change 1 of 4th fix for ticket #861 --- .../conf/type/__apt_update_index/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 70b59710..2d7f9030 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,9 +18,23 @@ # along with cdist. If not, see . # + +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists +# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get update || apt-get update + apt-get $apt_opts update || apt-get $apt_opts update fi DONE From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:22:16 +0200 Subject: [PATCH 083/128] __package_apt: fix complain about suite change the last fix for ticket #861 :-) --- cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index fbfca330..79c0d9d3 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get update > /dev/null 2>&1 || true +then echo apt-get $apt_opts update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 16 Sep 2021 21:36:39 +0300 Subject: [PATCH 084/128] add comments, add -u to diff --- cdist/conf/type/__sed/gencode-remote | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index be7690e1..f99c5a88 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -14,6 +14,8 @@ then script="$( cat "$__object/stdin" )" fi +# since stdin is not available in explorer, we pull file from target with explorer + file_from_target="$__object/explorer/file" sed_cmd='sed' @@ -23,10 +25,10 @@ then sed_cmd="$sed_cmd -E" fi -if echo "$script" \ - | "$sed_cmd" -f - "$file_from_target" \ - | diff "$file_from_target" - \ - > /dev/null +# do sed dry run, diff result and if no change, then there's nothing to do +# also redirect diff's output to stderr for debugging purposes + +if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 then exit 0 fi From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 21 Sep 2021 08:55:54 +0200 Subject: [PATCH 085/128] ++changelog --- docs/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog b/docs/changelog index 693d028f..ab637f5f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,11 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) + * New type: __sed (Ander Punnar) + * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __apt_source: Fix complaint about suite change (Matthias Stecher) + * Type __package_apt: Fix complaint about suite change (Matthias Stecher) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:06:45 +0200 Subject: [PATCH 086/128] [__debconf_set_selections] Fix --file not being supported Even if deprecated, the parameter *must* be supported, which isn't the case right now. This was due to a misunderstanding of how deprecating parameters work, see: https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters --- .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 + .../__debconf_set_selections/parameter/{deprecated => optional} | 0 2 files changed, 1 insertion(+) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%) diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file new file mode 100644 index 00000000..09db545a --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file @@ -0,0 +1 @@ +'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated rename to cdist/conf/type/__debconf_set_selections/parameter/optional From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:09:42 +0200 Subject: [PATCH 087/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ab637f5f..142602c8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) + * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:14:57 +0200 Subject: [PATCH 088/128] Improve bullseye support, perticularly __letsencrypt_cert --- cdist/conf/type/__grafana_dashboard/manifest | 2 +- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index d145c4c3..0d944482 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*) + 10*|11*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 6394f629..638a99e0 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*) + 10*|11*) __package_apt certbot ;; From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:16:11 +0200 Subject: [PATCH 089/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 142602c8..6f717cf4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) + * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 31 Oct 2021 17:38:10 +0100 Subject: [PATCH 090/128] [__haproxy_dualstack] New type with PROXY protocol support This is backwards compatible with what is already used internally @ungleich, but adds on top of that the ability to customise ports and, most importantly, it adds PROXY protocol support. --- .../conf/type/__haproxy_dualstack/files/http | 8 + .../conf/type/__haproxy_dualstack/files/https | 10 ++ .../conf/type/__haproxy_dualstack/files/imaps | 12 ++ .../conf/type/__haproxy_dualstack/files/smtps | 12 ++ cdist/conf/type/__haproxy_dualstack/man.rst | 121 ++++++++++++++ cdist/conf/type/__haproxy_dualstack/manifest | 155 ++++++++++++++++++ .../parameter/default/protocol | 1 + .../parameter/optional_multiple | 3 + cdist/conf/type/__haproxy_dualstack/singleton | 0 9 files changed, 322 insertions(+) create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http new file mode 100644 index 00000000..0508a465 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/http @@ -0,0 +1,8 @@ +frontend http + bind BIND@:80 + mode http + option httplog + default_backend http + +backend http + mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https new file mode 100644 index 00000000..73deac46 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/https @@ -0,0 +1,10 @@ +frontend https + bind BIND@:443 + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend https + +backend https + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps new file mode 100644 index 00000000..b1ec3793 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/imaps @@ -0,0 +1,12 @@ +frontend imaps + bind BIND@:143 + bind BIND@:993 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend imaps + +backend imaps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps new file mode 100644 index 00000000..dce6ed4a --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/smtps @@ -0,0 +1,12 @@ +frontend smtps + bind BIND@:25 + bind BIND@:465 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend smtps + +backend smtps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst new file mode 100644 index 00000000..6c131cbe --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/man.rst @@ -0,0 +1,121 @@ +cdist-type__haproxy_dualstack(7) +================================ + + +NAME +---- +cdist-type__haproxy_dualstack - Proxy services from a dual-stack server + + +DESCRIPTION +----------- +This (singleton) type installs and configures haproxy to act as a dual-stack +proxy for single-stack services. + +This can be useful to add IPv4 support to IPv6-only services while only using +one IPv4 for many such services. + +By default this type uses the plain TCP proxy mode, which means that there is no +need for TLS termination on this host when SNI is supported. +This also means that proxied services will not receive the client's IP address, +but will see the proxy's IP address instead (that of `$__target_host`). + +This can be solved by using the PROXY protocol, but do take into account that, +e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same +port, so you will need to use other ports for that. + +As a recommendation in this type: use TCP ports 8080 and 591 respectively to +serve HTTP and HTTPS using the PROXY protocol. + +See the EXAMPLES for more details. + + +OPTIONAL PARAMETERS +------------------- +v4proxy + Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. + In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is + the IP address actually providing the proxied services. + The full format of this argument is: + `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` + Where starting with `proxy:` determines that the PROXY protocol must be + used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT + override for the given PROTOCOL (see `--protocol`), if not present the + PROTOCOL's default port will be used. + + +v6proxy + Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. + In its simplest use, it must be a NAME with an `A` DNS entry, which is + the IP address actually providing the proxied services. + See `--v4proxy` for more options and details. + +protocol + Can be passed multiple times or as a space-separated list of protocols. + Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. + This defaults to: `http https imaps smtps`. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Proxy the IPv6-only services so IPv4-only clients can access them + # This uses HAProxy's TCP mode for http, https, imaps and smtps + __haproxy_dualstack \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them + # Note this means that the backend IPv6-only server will only see + # the IPv6 address of the haproxy host managed by cdist, which can be + # troublesome if this information is relevant for analytics/security/... + # See the PROXY example below + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling + # IPv4-only clients to access them while maintaining the client's IP address + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy proxy:ipv6.chat:http=8080:https=591 \ + --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 + # Note however that the PROXY protocol is not compatible with regular + # HTTP(S) protocols, so your nginx will have to listen on different ports + # with the PROXY settings. + # Note that you will need to restrict access to the 8080 port to prevent + # Client IP spoofing. + # This can be something like: + # server { + # # listen for regular HTTP connections + # listen [::]:80 default_server; + # listen 80 default_server; + # # listen for PROXY HTTP connections + # listen [::]:8080 proxy_protocol; + # # Accept the Client's IP from the PROXY protocol + # real_ip_header proxy_protocol; + # } + + +SEE ALSO +-------- +- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ +- https://www.haproxy.com/blog/haproxy/proxy-protocol/ +- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ + + +AUTHORS +------- +ungleich +Evilham + + +COPYING +------- +Copyright \(C) 2021 ungleich glarus ag. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest new file mode 100644 index 00000000..d110eea6 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/manifest @@ -0,0 +1,155 @@ +#!/bin/sh -eu + +__package haproxy +require="__package/haproxy" __start_on_boot haproxy + +tmpdir="$__object/files" +mkdir "$tmpdir" +configtmp="$__object/files/haproxy.cfg" + +os=$(cat "$__global/explorer/os") +case $os in + freebsd) + CONFIG_FILE="/usr/local/etc/haproxy.conf" + cat < "$configtmp" +global + maxconn 4000 + user nobody + group nogroup + daemon + +EOF + + ;; + *) + CONFIG_FILE="/etc/haproxy/haproxy.cfg" + cat < "$configtmp" +global + log [::1] local2 + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + +EOF + ;; +esac + +cat <> "$configtmp" +defaults + retries 3 + log global + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s +EOF + +dig_cmd="$(command -v dig || true)" +get_ip() { + # Usage: get_ip (ipv4|ipv6) NAME + # uses "dig" if available, else fallback to "host" + case $1 in + ipv4) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short A "$2" + else + host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' + fi + ;; + ipv6) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short AAAA "$2" + else + host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' + fi + ;; + esac +} + +PROTOCOLS="$(cat "$__object/parameter/protocol")" + +for proxy in v4proxy v6proxy; do + param=$__object/parameter/$proxy + # no backend? skip generating code + if [ ! -f "$param" ]; then + continue + fi + + # turn backend name into bind parameter: v4backend -> ipv4@ + bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') + + case $bind in + ipv4) + backendproto=ipv6 + ;; + ipv6) + backendproto=ipv4 + ;; + esac + + for proto in ${PROTOCOLS}; do + # Add protocol "header" + printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" + + sed -e "s/BIND/$bind/" \ + -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ + -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ + "$__type/files/$proto" >> "$configtmp" + + while read -r hostdefinition; do + if echo "$hostdefinition" | grep -qE '^proxy:'; then + # Proxy protocol was requested + host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" + send_proxy=" send-proxy" + else + # Just use tcp proxy mode + host="$hostdefinition" + send_proxy="" + fi + if echo "$hostdefinition" | grep -qE ":${proto}="; then + # Use custom port definition if requested + port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" + else + # Else use the default + port="" + fi + servername=$host + + res=$(get_ip "$bind" "$servername") + + if [ -z "$res" ]; then + echo "$servername does not resolve - aborting config" >&2 + exit 1 + fi + + # Treat protocols without TLS+SNI specially + if [ "$proto" = http ]; then + echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" + else + echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" + fi + + # Create the "server" itself. + # Note that port and send_proxy will be empty unless + # they were requested by the type user + echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" + + done < "$param" + done +done + +# Create config file +require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 + +require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ + --pattern "^__file${CONFIG_FILE}" \ + --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol new file mode 100644 index 00000000..dc8bb7bf --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol @@ -0,0 +1 @@ +http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple new file mode 100644 index 00000000..8c482bd4 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple @@ -0,0 +1,3 @@ +protocol +v4proxy +v6proxy diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton new file mode 100644 index 00000000..e69de29b From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001 From: Evilham Date: Wed, 3 Nov 2021 11:03:33 +0100 Subject: [PATCH 091/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6f717cf4..99a8c08b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) + * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 16 Nov 2021 11:11:45 +0100 Subject: [PATCH 092/128] Python 3.10: collections.X -> collections.abc.X --- cdist/integration.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/integration.py b/cdist/integration.py index 17b65f09..04470ea7 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.Iterable): + elif isinstance(host, collections.abc.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 09e9cc19..6bf935e8 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.MutableSequence): +class FileList(collections.abc.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.MutableSequence): self.__write(lines) -class DirectoryDict(collections.MutableMapping): +class DirectoryDict(collections.abc.MutableMapping): """A dict that stores it's items as files in a directory. """ From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:07:28 +0100 Subject: [PATCH 093/128] [__package_upgrade_all] Add new --apt-with-new-pkgs argument --- cdist/conf/type/__package_upgrade_all/gencode-remote | 6 +++++- cdist/conf/type/__package_upgrade_all/man.rst | 8 ++++++++ cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 38aa001e..d332e851 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" +if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then + apt_with_new_pkgs="--with-new-pkgs" +fi + if [ -f "$type" ]; then type="$(cat "$type")" else @@ -54,7 +58,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget upgrade" + else echo "$aptget $apt_with_new_pkgs upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index e9e2b8ce..0c116bac 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,6 +33,14 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. +apt-with-new-pkg + Allow installing new packages when used in conjunction with + upgrade. This is useful if the update of an installed package + requires new dependencies to be installed. Instead of holding the + package back upgrade will upgrade the package and install the new + dependencies. Note that upgrade with this option will never remove + packages, only allow adding new ones. + apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index 7a56a34b..cd22eb90 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,2 +1,3 @@ apt-clean apt-dist-upgrade +apt-with-new-pkgs From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:08:49 +0100 Subject: [PATCH 094/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 99a8c08b..26d89057 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Tue, 8 Mar 2022 12:04:58 +0100 Subject: [PATCH 095/128] Added rm of tmpfile. --- cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 61c77fb9..cbffde94 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,6 +40,7 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" +rm -f "\$tmpfile" DONE } From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Mar 2022 16:16:44 +0100 Subject: [PATCH 096/128] ++changes --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 26d89057..81be51f6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,8 @@ next: * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) + * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) + * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 11:22:31 +0100 Subject: [PATCH 097/128] Fix typos; add default priority; comments in generated files --- cdist/conf/type/__apt_pin/manifest | 5 +++++ cdist/conf/type/__apt_pin/parameter/default/priority | 1 + cdist/conf/type/__apt_pin/parameter/optional | 1 + cdist/conf/type/__apt_pin/parameter/required | 1 - 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index e72a8fdd..983b2b42 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF +# Created by cdist ${__type##*/} +# Do not change. Changes will be overwritten. +# + +# $name Package: $package Pin: $pin Pin-Priority: $priority diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority new file mode 100644 index 00000000..1b79f38e --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/priority @@ -0,0 +1 @@ +500 diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional index 52f01fd2..847e703d 100644 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -1,2 +1,3 @@ state package +priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required index 4b4e9741..c8572d92 100644 --- a/cdist/conf/type/__apt_pin/parameter/required +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -1,2 +1 @@ distribution -priority From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sun, 10 Apr 2022 23:52:53 +0200 Subject: [PATCH 098/128] __file: make file uploading and attribute changes more atomic Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++++-------- cdist/conf/type/__file/gencode-remote | 14 +++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 231b6927..bea3d79c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - tempfile_template="${destination}.cdist.XXXXXXXXXX" + upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + # Yes, we are aware that this is a race condition. + # However: + # a) cdist usually writes to directories that are not user writable + # (probably > 99.9%) + # b) if they are user owned, the user / attacker always wins + # (probably < 0.1%) + # c) the only case which we could improve are tmp directories and we + # don't think managing tmp directories with cdist is a typical case + # ("the rest %)" cat << DONE -destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +$__remote_exec $__target_host test -e $upload_destination && { + echo "Refusing to upload file to existing destination: $upload_destination" >&2 + exit 1 +} DONE + # Tell gencode-remote to where we uploaded the file so it can move + # it to its final destination. + echo "$upload_destination" > "$__object/files/upload-destination" + if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix @@ -103,12 +119,8 @@ DONE my_target_host="${__target_host}" fi cat << DONE -$__remote_copy "$source" "${my_target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE fi -# move uploaded file into place -cat << DONE -$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" -DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index f7a528fd..136520a7 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,6 +62,13 @@ set_mode() { case "$state_should" in present|exists) + if [ -f "$__object/files/upload-destination" ]; then + final_destination="$destination" + # We change the 'global' $destination variable here so we can + # change attributes of the new/uploaded file before moving it + # to it's final destination. + destination="$(cat "$__object/files/upload-destination")" + fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -81,6 +88,11 @@ case "$state_should" in fi fi done + if [ -f "$__object/files/upload-destination" ]; then + # move uploaded file into place + printf 'rm -rf "%s"\n' "$final_destination" + printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 From af54fe6febc186b0db6da00f2d06d5e87f4012f9 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 11 Apr 2022 00:04:41 +0200 Subject: [PATCH 099/128] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 81be51f6..3363be27 100644 --- a/docs/changelog +++ b/docs/changelog @@ -14,6 +14,7 @@ next: * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) + * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From cb0fa0f2e4d99891653b8871309e22f62ff393dd Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:39:29 +0100 Subject: [PATCH 100/128] force add-apt-repository to act in non-interactive mode --- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 84ebebfe..094366b9 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,7 +29,7 @@ fi case "$state_should" in present) - echo "add-apt-repository '$name'" + echo "add-apt-repository -y '$name'" ;; absent) echo "remove-apt-repository '$name'" From 54a5cb17b762a7fbb61225b36621c8a45e83536b Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:41:03 +0100 Subject: [PATCH 101/128] use add-apt-repository instead of add-apt-repository Remove `remove-apt-repository` which is now no longer needed; use `add-apt-repository` which allow removal through the `-r` flag. --- .../__apt_ppa/files/remove-apt-repository | 55 ------------------- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- cdist/conf/type/__apt_ppa/manifest | 5 -- 3 files changed, 1 insertion(+), 61 deletions(-) delete mode 100755 cdist/conf/type/__apt_ppa/files/remove-apt-repository diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository deleted file mode 100755 index 3eb7d491..00000000 --- a/cdist/conf/type/__apt_ppa/files/remove-apt-repository +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Remove the given apt repository. -# -# Exit with: -# 0: if it worked -# 1: if not -# 2: on other error - -import os -import sys -from aptsources import distro, sourceslist -from softwareproperties import ppa -from softwareproperties.SoftwareProperties import SoftwareProperties - - -def remove_if_empty(file_name): - with open(file_name, 'r') as f: - if f.read().strip(): - return - os.unlink(file_name) - -def remove_repository(repository): - #print 'repository:', repository - codename = distro.get_distro().codename - #print 'codename:', codename - (line, file) = ppa.expand_ppa_line(repository.strip(), codename) - #print 'line:', line - #print 'file:', file - deb_source_entry = sourceslist.SourceEntry(line, file) - src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) - - try: - sp = SoftwareProperties() - sp.remove_source(deb_source_entry) - try: - # If there's a deb-src entry, remove that too - sp.remove_source(src_source_entry) - except: - pass - remove_if_empty(file) - return True - except ValueError: - print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line - return False - -if __name__ == '__main__': - if (len(sys.argv) != 2): - print >> sys.stderr, 'Error: need a repository as argument' - sys.exit(2) - repository = sys.argv[1] - if remove_repository(repository): - sys.exit(0) - else: - sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 094366b9..e41341b8 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -32,6 +32,6 @@ case "$state_should" in echo "add-apt-repository -y '$name'" ;; absent) - echo "remove-apt-repository '$name'" + echo "add-apt-repository -r -y '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index c6f4e876..57e85442 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,9 +20,4 @@ __package software-properties-common -require="__package/software-properties-common" \ - __file /usr/local/bin/remove-apt-repository \ - --source "$__type/files/remove-apt-repository" \ - --mode 0755 - require="$__object_name" __apt_update_index From 6f8c774cb08e29c8f0282a8a40a43f1d09c7248e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:16:10 +0200 Subject: [PATCH 102/128] workaround mktemp -u checking for write access Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index bea3d79c..42c9d1e7 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -89,7 +89,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 2df2578e36c688fcbdb32a87609bc263d2e7db9c Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:27:28 +0200 Subject: [PATCH 103/128] __file: remove the questionable check for uploadfile existence Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 42c9d1e7..aa7149c1 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -99,12 +99,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # c) the only case which we could improve are tmp directories and we # don't think managing tmp directories with cdist is a typical case # ("the rest %)" - cat << DONE -$__remote_exec $__target_host test -e $upload_destination && { - echo "Refusing to upload file to existing destination: $upload_destination" >&2 - exit 1 -} -DONE + # Tell gencode-remote to where we uploaded the file so it can move # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" From 8b915b15b5a969b7de308e2b1b124d28548349dc Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:46:13 +0200 Subject: [PATCH 104/128] __file: make the create-empty-file case work again Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index aa7149c1..f9acdb17 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -72,6 +72,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 + echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -88,6 +89,14 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" + if [ "$create_file" ]; then + # When creating an empty file we create it locally and then + # upload it so that permissions can be set before moving the file + # into place. + source="$__object/files/empty" + touch "$source" + fi + # upload file to temp location upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. @@ -104,18 +113,15 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" - if [ "$upload_file" ]; then - echo upload >> "$__messages_out" - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi - cat << DONE + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE $__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE - fi fi fi From abbc7dfc376ffb6554bfaad2d5a6ec93955f235f Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 16 Apr 2022 19:05:31 +0200 Subject: [PATCH 105/128] since we already remove the destination, we have no need to use -T on move, fixes #333 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 136520a7..1a9ff69c 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -91,7 +91,7 @@ case "$state_should" in if [ -f "$__object/files/upload-destination" ]; then # move uploaded file into place printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + printf 'mv "%s" "%s"\n' "$destination" "$final_destination" fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local From 6c8c692a22c886bf82b18f51a133f88a8ab547b7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 2 May 2022 23:25:59 +0200 Subject: [PATCH 106/128] __file: kiss and fix regression on Mac OSX Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index f9acdb17..5a303308 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -98,7 +98,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then fi # upload file to temp location - upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" + upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 3d58c9b24fede2d1dafb15fc36424c52c466bb0d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Fri, 20 May 2022 13:48:07 +0200 Subject: [PATCH 107/128] add optional file parameter to allow for use in a loop without object_id clashes --- cdist/conf/type/__dot_file/man.rst | 15 +++++++++++++++ cdist/conf/type/__dot_file/manifest | 10 ++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index ba7621a1..c8f36712 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,6 +37,12 @@ state source forwarded to :strong:`__file` type +file + forwarded to :strong:`__file` type + This can be used if multiple users need to have a dotfile updated, + which will result in duplicate object id errors. When using the + file parameter the object id can be some unique value. + MESSAGES -------- @@ -61,6 +67,15 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" + # install .vimrc for root and some users + for user in root userx usery userz; do + __dot_file "${user}_dot_vimrc" \ + --user $user \ + --file .vimrc \ + --state exists \ + --source "$__files/$user/.vimrc" + done + SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 02dadf05..a38ed943 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,13 +20,19 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" +if [ -f "${__object}/parameter/file" ]; then + file="$(cat "${__object}/parameter/file")" +else + file="${__object_id}" +fi + # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${__object_id} +subpath=${file} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -64,4 +70,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" From 9839c2d8ec63719c30493870851f1e1ab8079fb1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:55:12 +0200 Subject: [PATCH 108/128] ++changelog Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3363be27..29507443 100644 --- a/docs/changelog +++ b/docs/changelog @@ -15,6 +15,7 @@ next: * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) + * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 77d9a757ec0e6986388aa368b77f065ae6be68c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:58:45 +0200 Subject: [PATCH 109/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 29507443..8c9e1028 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) + * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 0ae37b3445c6662a968c49972e0102c75d6f14c2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 1 Jul 2022 14:27:33 +0100 Subject: [PATCH 110/128] Handle signed-by option in __apt_source Allow users to specify a GPG key fingerprint or keyring file to be included as the 'signed-by' option. --- .../type/__apt_source/files/source.list.template | 3 ++- cdist/conf/type/__apt_source/man.rst | 8 ++++++++ cdist/conf/type/__apt_source/manifest | 14 ++++++++++---- cdist/conf/type/__apt_source/parameter/optional | 3 ++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index d4420e96..a28bb45f 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,13 +2,14 @@ set -u entry="$uri $distribution $component" + cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${forcedarch} $entry +deb ${options} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d1acb388..d317a135 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,6 +23,9 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) +signed-by + provide a GPG key fingerprint or keyring path for signature checks + state 'present' or 'absent', defaults to 'present' @@ -56,6 +59,11 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present + __apt_source goaccess \ + --uri http://deb.goaccess.io/ \ + --component main \ + --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D + AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index 35f15909..cdb526d3 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -31,9 +31,15 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - forcedarch="[arch=$(cat "$__object/parameter/arch")]" -else - forcedarch="" + options="arch=$(cat "$__object/parameter/arch")" +fi + +if [ -f "$__object/parameter/signed-by" ]; then + options="$options signed-by=$(cat "$__object/parameter/signed-by")" +fi + +if [ "$options" ]; then + options="[$options]" fi # export variables for use in template @@ -41,7 +47,7 @@ export name export uri export distribution export component -export forcedarch +export options # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 87537335..0b5470a1 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,4 +1,5 @@ state distribution component -arch \ No newline at end of file +arch +signed-by From 339ca9347b8dc3d9167393bf754a92c4dedbd704 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 2 Jul 2022 19:21:27 +0200 Subject: [PATCH 111/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8c9e1028..3d9cb704 100644 --- a/docs/changelog +++ b/docs/changelog @@ -17,6 +17,7 @@ next: * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) + * Type __apt_source: Add signed-by parameter (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 483f0c161440a018efe48bc96910dcb5834c32a1 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Wed, 13 Jul 2022 14:50:17 +0200 Subject: [PATCH 112/128] add Check Point Gaia --- cdist/conf/explorer/lsb_codename | 3 +++ cdist/conf/explorer/lsb_description | 3 +++ cdist/conf/explorer/lsb_id | 3 +++ cdist/conf/explorer/lsb_release | 3 +++ cdist/conf/explorer/os | 7 +++++++ cdist/conf/explorer/os_release | 4 ++++ cdist/conf/explorer/os_version | 3 +++ 7 files changed, 26 insertions(+) diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index 26bb8e3d..c9fb5cdf 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + awk '{printf("%s\n", $(NF-1))}' /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index b1009627..7279a9c2 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 82ff9977..1f91cc40 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + echo "CheckPoint" + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 5ebfff1a..d9100569 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 46d87f3e..b9232ee4 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,6 +116,13 @@ if [ -f /etc/slackware-version ]; then exit 0 fi +# Appliances + +if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then + echo checkpoint + exit 0 +fi + uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index 6489446b..ec85046f 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,5 +34,9 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release +elif test -f /etc/cp-release +then + # Checkpoint firewall or management (actually linux based) + cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index bbc9e4f0..430200ae 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -41,6 +41,9 @@ in # empty, but well... cat /etc/arch-release ;; + checkpoint) + awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release + ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version From 624316564531cc7575d2ee0504e3a5f684c74e3d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 16:27:12 +0200 Subject: [PATCH 113/128] add create and ifexists to line type --- cdist/conf/type/__line/gencode-remote | 12 ++++++++++-- cdist/conf/type/__line/man.rst | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index a89886da..95c2360d 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,8 +34,16 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 + if [ -f "$__object/parameter/create" ]; then + echo "touch $file" + else + # only bark if the file should exists + if [ -f "$__object/parameter/ifexists" ]; then + exit 0 + fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 + fi fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index 70490f68..dd41fc40 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,6 +21,9 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. +create + It the file does not exist then create an empty file + before Insert the given line before this pattern. @@ -28,6 +31,9 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. +ifexists + Only apply the line if the file exists. + line Specifies the line which should be absent or present. From 7d8fc8a5c38c47a36ea87cd98d798d22f91c3185 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:18:41 +0200 Subject: [PATCH 114/128] improve checkpoint sed, add __line changes --- cdist/conf/explorer/lsb_release | 2 +- cdist/conf/type/__line/parameter/boolean | 2 ++ cdist/conf/type/__line/parameter/default/create | 1 + cdist/conf/type/__line/parameter/default/ifexists | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__line/parameter/boolean create mode 100644 cdist/conf/type/__line/parameter/default/create create mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index d9100569..0bb9f7fe 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -22,7 +22,7 @@ set +e case "$("$__explorer/os")" in checkpoint) - cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' ;; openwrt) # shellcheck disable=SC1091 diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean new file mode 100644 index 00000000..182a5da6 --- /dev/null +++ b/cdist/conf/type/__line/parameter/boolean @@ -0,0 +1,2 @@ +create +ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/create @@ -0,0 +1 @@ +false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/ifexists @@ -0,0 +1 @@ +false From 17466452f0762fc3a259a66742a59943c5af2510 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:53:41 +0200 Subject: [PATCH 115/128] revert __line for clean PR history --- cdist/conf/type/__line/gencode-remote | 12 ++---------- cdist/conf/type/__line/man.rst | 6 ------ cdist/conf/type/__line/parameter/boolean | 2 -- cdist/conf/type/__line/parameter/default/create | 1 - cdist/conf/type/__line/parameter/default/ifexists | 1 - 5 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 cdist/conf/type/__line/parameter/boolean delete mode 100644 cdist/conf/type/__line/parameter/default/create delete mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 95c2360d..a89886da 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,16 +34,8 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - if [ -f "$__object/parameter/create" ]; then - echo "touch $file" - else - # only bark if the file should exists - if [ -f "$__object/parameter/ifexists" ]; then - exit 0 - fi - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 - fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index dd41fc40..70490f68 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,9 +21,6 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. -create - It the file does not exist then create an empty file - before Insert the given line before this pattern. @@ -31,9 +28,6 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. -ifexists - Only apply the line if the file exists. - line Specifies the line which should be absent or present. diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean deleted file mode 100644 index 182a5da6..00000000 --- a/cdist/conf/type/__line/parameter/boolean +++ /dev/null @@ -1,2 +0,0 @@ -create -ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/create +++ /dev/null @@ -1 +0,0 @@ -false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/ifexists +++ /dev/null @@ -1 +0,0 @@ -false From d4bf41ce3b7d186e7e85c030c5960ddd38855a29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Jul 2022 10:57:01 +0200 Subject: [PATCH 116/128] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3d9cb704..db2faef0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -18,6 +18,7 @@ next: * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) * Type __apt_source: Add signed-by parameter (Daniel Fancsali) + * Explorer: add support for checkpoint (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From be6e7fcc08d5c4e98f7ce30b157ee087010cd126 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 31 Jul 2022 21:59:35 +0200 Subject: [PATCH 117/128] Prepare release of cdist 7.0.0 --- bin/cdist-build-helper | 2 +- docs/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index 0380b3f8..cadddae7 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. diff --git a/docs/changelog b/docs/changelog index db2faef0..00defc2a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) * New type: __haproxy_dualstack (Evilham and ungleich) From 90488d2e9e21b4e6bf3b7f0d8c44ebfe29c517b7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Aug 2022 00:03:51 +0200 Subject: [PATCH 118/128] [doc] add release process documentation --- docs/dev/release-process.org | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/dev/release-process.org diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org new file mode 100644 index 00000000..42b4f5c5 --- /dev/null +++ b/docs/dev/release-process.org @@ -0,0 +1,90 @@ +* Install requirements (Alpine) + - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ + py3-build twine +* Ensure your gpg setup works with the email used in the git commit! + - For me this is nico@nico-notebook.schottelius.org + - Signature / id is on nb2 +* Create ~/.pypirc +[distutils] + index-servers = + pypi + cdist + +[pypi] + username = __token__ + password = ... + +[cdist] + repository = https://upload.pypi.org/legacy/ + username = __token__ + password = ... + +* Add date in docs/changelog +* Run ./bin/cdist-build-helper +* TODO Move to "build" + - python3 -m build +* DONE git tag: when? +CLOSED: [2022-07-31 Sun 23:58] +** Asked during release process: ok +* DONE Pypi error with distutils: do not use distutils anymore +CLOSED: [2022-07-31 Sun 23:58] +python3 setup.py sdist upload +... +Creating tar archive +removing 'cdist-7.0.0' (and everything under it) +running upload +Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ +Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +(venv2) [22:50] nb2:cdist% + +* DONE Pypi error with twine: fixed in twine 4.0.1 +CLOSED: [2022-07-31 Sun 23:58] + +Seeing: + +(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* +Uploading distributions to https://upload.pypi.org/legacy/ +Traceback (most recent call last): + File "/usr/bin/twine", line 8, in + sys.exit(main()) + File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main + result = cli.dispatch(sys.argv[1:]) + File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch + return main(args.args) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main + return upload(upload_settings, parsed_args.dists) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload + resp = repository.upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload + resp = self._upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload + data = package.metadata_dictionary() + File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary + "dynamic": meta.dynamic, +AttributeError: 'Wheel' object has no attribute 'dynamic' + + +Fix: + + +(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* +⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. +Uploading distributions to https://upload.pypi.org/legacy/ +Uploading cdist-7.0.0-py3-none-any.whl +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s +Uploading cdist-7.0.0.tar.gz +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s + +View at: +https://pypi.org/project/cdist/7.0.0/ +* TODO cdist web + - on staticweb-2022 + - Should be moved to sftp/k8s + + + Manual steps: + + ~/bin/permissions.public html/ + rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ + ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest" From c85184dcb48118ea5edbda2799badb5779253f7c Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Sun, 18 Sep 2022 08:49:37 +0200 Subject: [PATCH 119/128] Make sure flag is followed by end of line or space. --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 00646c75..0f212e73 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -515,7 +515,7 @@ check_vm_arch_specific() { && return 0 fi if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\) /proc/cpuinfo then return 0 fi From 62db96bb376354c829cdde290c915cffc24612b0 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Thu, 29 Sep 2022 16:19:07 +0200 Subject: [PATCH 120/128] Initialise options variable so expansion when running files/source.list.template there will not be an error when the variable is not set. --- cdist/conf/type/__apt_source/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index cdb526d3..fd1ec47f 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -21,6 +21,7 @@ name="$__object_id" state="$(cat "$__object/parameter/state")" uri="$(cat "$__object/parameter/uri")" +options="" if [ -f "$__object/parameter/distribution" ]; then distribution="$(cat "$__object/parameter/distribution")" From ffeaafe9b68a944418ec8496c7db9f347dd4b1ee Mon Sep 17 00:00:00 2001 From: marcoduif Date: Fri, 7 Oct 2022 07:22:31 +0000 Subject: [PATCH 121/128] Make grep more specific package name should be an exact match, not a substring --- cdist/conf/type/__apt_mark/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state index b7fe08fa..b464179a 100755 --- a/cdist/conf/type/__apt_mark/explorer/state +++ b/cdist/conf/type/__apt_mark/explorer/state @@ -24,4 +24,4 @@ else name="$__object_id" fi -apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold +apt-mark showhold | grep -q "^${name}$" && echo hold || echo unhold From f36069754c0101d1eccbef59fa4f060a19784d26 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Dec 2022 18:03:15 +0100 Subject: [PATCH 122/128] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 00defc2a..af41368d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +7.0.1: + * Type __apt_mark: Narrow down grep for hold packages (marcoduif) + 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) From bdfd92dc3707004503af4a03aea0819ad587a947 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 21 Dec 2022 09:41:33 +0100 Subject: [PATCH 123/128] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index af41368d..e8090131 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog 7.0.1: * Type __apt_mark: Narrow down grep for hold packages (marcoduif) + * Type __apt_source: Set required options variable (Mark Verboom) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From b974969f28f4d007f75904757e80e4f663e3d134 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Dec 2022 20:59:16 +0100 Subject: [PATCH 124/128] Remove double definition of scan parser Fixes #353 --- cdist/argparse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f17315e7..8f7bbb85 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -472,9 +472,6 @@ def get_parsers(): parser['info'].set_defaults(func=cdist.info.Info.commandline) # Scan = config + further - parser['scan'] = parser['sub'].add_parser('scan', add_help=False, - parents=[parser['config']]) - parser['scan'] = parser['sub'].add_parser( 'scan', parents=[parser['loglevel'], parser['beta'], From ed3da3c829d47c7c578733482920f111e8bc8869 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Dec 2022 21:02:41 +0100 Subject: [PATCH 125/128] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e8090131..6ade5d47 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog 7.0.1: * Type __apt_mark: Narrow down grep for hold packages (marcoduif) * Type __apt_source: Set required options variable (Mark Verboom) + * Core: Remove double definition of scan parser (Nico Schottelius) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From 08a6b467fa7afcd2f2a0a2497441cee773166d6e Mon Sep 17 00:00:00 2001 From: Michelle Date: Wed, 25 Jan 2023 16:06:35 -0500 Subject: [PATCH 126/128] Added support for Devuan Daedalus Added one line that allows cdist to support Devuan Daedelus version --- cdist/conf/explorer/os_version | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 430200ae..fc59fd14 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -82,6 +82,7 @@ in # ceres versions don't have a number, so we decode by codename: case ${devuan_version} in + (daedalus/ceres) echo 4.99 ;; (chimaera/ceres) echo 3.99 ;; (beowulf/ceres) echo 2.99 ;; (ascii/ceres) echo 1.99 ;; From 1450861e26c8c5dd249517fa14fa8ea0f4961d3d Mon Sep 17 00:00:00 2001 From: Michelle Date: Thu, 2 Feb 2023 18:21:43 -0500 Subject: [PATCH 127/128] Updated the python version of cerbot freebsd The package referenced for cerbot to be install in the FreeBSD platform used python 3.7 package, updated to python 3.9 --- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 638a99e0..39067f3b 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -85,7 +85,7 @@ if [ -z "${certbot_fullpath}" ]; then esac ;; freebsd) - __package py37-certbot + __package py39-certbot certbot_fullpath="/usr/local/bin/certbot" ;; ubuntu) From 7dd2d1025ae3b005e5d44d11e1ca62536119f36c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 3 Feb 2023 22:54:13 +0100 Subject: [PATCH 128/128] ++changelog --- docs/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 6ade5d47..4e9a8a6d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,9 +2,12 @@ Changelog --------- 7.0.1: + * Core: Remove double definition of scan parser (Nico Schottelius) * Type __apt_mark: Narrow down grep for hold packages (marcoduif) * Type __apt_source: Set required options variable (Mark Verboom) - * Core: Remove double definition of scan parser (Nico Schottelius) + * Type __letsencrypt_cert: Update python version (Michelle) + * Explorer os_version: Add support for Daedalus (Michelle) + * Explorer machine_type: Correct incorrect VMM matching (Mark Verboom) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera)