From 485283f2e531f5fa23a876fa52411e0030f32589 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 20:47:22 +0300 Subject: [PATCH 01/77] new type: __sed --- cdist/conf/type/__sed/explorer/file | 22 +++++++++ cdist/conf/type/__sed/gencode-remote | 46 +++++++++++++++++ cdist/conf/type/__sed/man.rst | 49 +++++++++++++++++++ cdist/conf/type/__sed/parameter/boolean | 1 + cdist/conf/type/__sed/parameter/optional | 1 + .../type/__sed/parameter/required_multiple | 1 + 6 files changed, 120 insertions(+) create mode 100755 cdist/conf/type/__sed/explorer/file create mode 100755 cdist/conf/type/__sed/gencode-remote create mode 100644 cdist/conf/type/__sed/man.rst create mode 100644 cdist/conf/type/__sed/parameter/boolean create mode 100644 cdist/conf/type/__sed/parameter/optional create mode 100644 cdist/conf/type/__sed/parameter/required_multiple diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file new file mode 100755 index 00000000..a40f71b3 --- /dev/null +++ b/cdist/conf/type/__sed/explorer/file @@ -0,0 +1,22 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +if [ -f "$file" ] +then + if [ -s "$file" ] + then + cat "$file" + else + echo "$file is empty" >&2 + exit 1 + fi +else + echo "$file do not exist" >&2 + exit 1 +fi diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote new file mode 100755 index 00000000..391e3bc1 --- /dev/null +++ b/cdist/conf/type/__sed/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/file" ] +then + file="$( cat "$__object/parameter/file" )" +else + file="/$__object_id" +fi + +script="$( cat "$__object/parameter/script" )" + +if [ "$script" = '-' ] +then + script="$( cat "$__object/stdin" )" +elif + [ -f "$script" ] +then + script="$( cat "$script" )" +fi + +file_from_target="$__object/explorer/file" + +sed_cmd='sed' + +if [ -f "$__object/parameter/regexp-extended" ] +then + sed_cmd="$sed_cmd --regexp-extended" +fi + +if ! echo "$script" \ + | "$sed_cmd" -f - "$file_from_target" \ + | diff "$file_from_target" - \ + > /dev/null +then + echo 'tmp="$( mktemp )"' + + echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + + echo "$script" + + echo 'EOF' + + echo "cp \"\$tmp\" '$file'" + + echo 'rm -f "$tmp"' +fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst new file mode 100644 index 00000000..fbac212d --- /dev/null +++ b/cdist/conf/type/__sed/man.rst @@ -0,0 +1,49 @@ +cdist-type__sed(7) +================== + +NAME +---- +cdist-type__sed - Transform text files with ``sed`` + + +DESCRIPTION +----------- +TODO + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +script + TODO + + +OPTIONAL PARAMETERS +------------------- +file + TODO + + +BOOLEAN PARAMETERS +------------------ +regexp-extended + TODO + + +EXAMPLES +-------- +.. code-block:: sh + + true + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean new file mode 100644 index 00000000..1ad75c5d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/boolean @@ -0,0 +1 @@ +regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional new file mode 100644 index 00000000..f73f3093 --- /dev/null +++ b/cdist/conf/type/__sed/parameter/optional @@ -0,0 +1 @@ +file diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple new file mode 100644 index 00000000..84f7e31d --- /dev/null +++ b/cdist/conf/type/__sed/parameter/required_multiple @@ -0,0 +1 @@ +script From 7a5896acfad0f992d6e7fe9cbe0bd08035ce19db Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:23:25 +0300 Subject: [PATCH 02/77] add --onchange, fix shellcheck --- cdist/conf/type/__sed/gencode-remote | 9 ++++++++- cdist/conf/type/__sed/man.rst | 3 +++ cdist/conf/type/__sed/parameter/optional | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 391e3bc1..6b1bd216 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -32,6 +32,7 @@ if ! echo "$script" \ | diff "$file_from_target" - \ > /dev/null then + # shellcheck disable=SC2016 echo 'tmp="$( mktemp )"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" @@ -41,6 +42,12 @@ then echo 'EOF' echo "cp \"\$tmp\" '$file'" - + + # shellcheck disable=SC2016 echo 'rm -f "$tmp"' + + if [ -f "$__object/parameter/onchange" ] + then + cat "$__object/parameter/onchange" + fi fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index fbac212d..4a728184 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -22,6 +22,9 @@ OPTIONAL PARAMETERS file TODO +onchange + TODO + BOOLEAN PARAMETERS ------------------ diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional index f73f3093..fa86f917 100644 --- a/cdist/conf/type/__sed/parameter/optional +++ b/cdist/conf/type/__sed/parameter/optional @@ -1 +1,2 @@ file +onchange From cf0032d667eb4bd8b7fe1bebacec341bbf71ed42 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 7 Jul 2021 21:28:00 +0300 Subject: [PATCH 03/77] add messaging and exit earlier --- cdist/conf/type/__sed/gencode-remote | 42 +++++++++++++++------------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 6b1bd216..836506f0 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -27,27 +27,31 @@ then sed_cmd="$sed_cmd --regexp-extended" fi -if ! echo "$script" \ +if echo "$script" \ | "$sed_cmd" -f - "$file_from_target" \ | diff "$file_from_target" - \ > /dev/null then - # shellcheck disable=SC2016 - echo 'tmp="$( mktemp )"' - - echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - - echo "$script" - - echo 'EOF' - - echo "cp \"\$tmp\" '$file'" - - # shellcheck disable=SC2016 - echo 'rm -f "$tmp"' - - if [ -f "$__object/parameter/onchange" ] - then - cat "$__object/parameter/onchange" - fi + exit 0 +fi + +# shellcheck disable=SC2016 +echo 'tmp="$( mktemp )"' + +echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" + +echo "$script" + +echo 'EOF' + +echo "cp \"\$tmp\" '$file'" + +# shellcheck disable=SC2016 +echo 'rm -f "$tmp"' + +echo 'change' >> "$__messages_out" + +if [ -f "$__object/parameter/onchange" ] +then + cat "$__object/parameter/onchange" fi From edcac70b2a99ce532dd5bca9a0b8fc5bf6dc5148 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 21 Jul 2021 13:22:34 +0200 Subject: [PATCH 04/77] [explorer/machine_type] Reimplement --- cdist/conf/explorer/machine_type | 986 ++++++++++++++++++++++++++++--- 1 file changed, 904 insertions(+), 82 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1c84f4d7..29f98849 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,8 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Daniel Heule (hda at sfs.biz) -# 2014 Thomas Oettli (otho at sfs.biz) -# 2020 Evilham (contact at evilham.com) +# 2021 Dennis Camera (cdist at dtnr.ch) # # This file is part of cdist. # @@ -19,91 +17,915 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to determine what type of machine the target to be +# configured is (container, virtual machine, bare-metal). +# +# It will print one line for each layer it can detect. +# The format of all lines is: TYPE[ VERB VENDOR] +# +# VERB does not have a special meaning, it is just for better readability. +# +# e.g. +# container on lxc +# virtual by kvm-spapr +# +# The third word of each line can be composed of different parts concatenated with a `-' +# (minus) character, with each component being a specification of the previous, +# e.g.: +# - lxc-libvirt (LXC container, managed by libvirt) +# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) +# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) +# +# If this explorer cannot determine any information it will print nothing. +# -os=$("$__explorer/os") +# Add /sbin and /usr/sbin to the path so we can find system +# binaries like dmidecode. +PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' +PATH="/sbin:/usr/sbin:${PATH}" +export PATH -vendor_string_to_machine_type() { - for vendor in vmware bochs kvm qemu virtualbox bhyve; do - if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then - vendor="kvm" - fi - echo "virtual_by_${vendor}" - exit - fi - done +arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') +uname_s=$(uname -s) + + +is_command() { command -v "$1" >/dev/null 2>&1; } + +is_oneof() ( + x=$1; shift + for y + do + test "${x}" = "${y}" || continue + return 0 + done + return 1 +) + +tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } + +# shellcheck disable=SC2086 +glob_exists() { set -- $1; test -e "$1"; } + +get_dmi_field() { + if is_oneof "${uname_s}" NetBSD + then + case $1 + in + (system-manufacturer) _mib=machdep.dmi.system-vendor ;; + (system-product-name) _mib=machdep.dmi.system-product ;; + (system-version|system-uuid) _mib=machdep.dmi.$1 ;; + (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; + (biod-release-date) _mib=machdep.dmi.bios-date ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + if is_command dmidecode + then + dmidecode -s "$1" + elif test -d "${dmi_sysfs-}" + then + case $1 + in + (system-manufacturer) _filename=sys_vendor ;; + (system-product-name) _filename=product_name ;; + (*) _filename=$(echo "$1" | tr - _) ;; + esac + if test -r "${dmi_sysfs-}/${_filename}" + then + cat "${dmi_sysfs}/${_filename}" + fi + unset _filename + elif test "${uname_s}" = OpenBSD + then + # NOTE: something similar to system-manufacutrer and system-product-name + # is available on OpenBSD in sysctl + case $1 + in + (system-manufacturer) _mib=hw.vendor ;; + (system-product-name) _mib=hw.product ;; + (*) _mib= ;; + esac + + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + return 1 } -case "$os" in - "freebsd") - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" - if [ -n "${vm_guest}" ]; then - if [ "${vm_guest}" = "none" ]; then - echo "physical" - exit - fi - echo "virtual_by_${vm_guest}" - exit - fi - ;; +has_cpuinfo() { test -e /proc/cpuinfo; } - "openbsd") - # OpenBSD can also use the sysctl's: hw.vendor or hw.product. - # Note we can be reasonably sure about a machine being virtualised - # as long as we can identify the virtualisation technology. - # But not so much about it being physical... - # Patches are welcome / reach out if you have better ideas. - for sysctl in hw.vendor hw.product; do - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" - done - ;; +get_sysctl() { + is_command sysctl && sysctl -n "$1" 2>/dev/null +} - *) - # Defaulting to linux for compatibility with previous cdist behaviour - if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit - fi +# Check for container - if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit - fi +has_ct_pid_1() { + test -r /run/systemd/container -o -r /proc/1/environ +} - if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm, bhyve - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - for vendor_file in /sys/class/dmi/id/product_name \ - /sys/class/dmi/id/sys_vendor \ - /sys/class/dmi/id/chasis_vendor; do - if [ -r ${vendor_file} ]; then - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(cat "${vendor_file}")" - fi - done - fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi - fi - ;; -esac +translate_container_name() { + case $1 + in + ('lxc') + echo lxc ;; + ('lxc-libvirt') + echo lxc-libvirt ;; + ('podman') + echo podman ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + return 1 ;; + esac + return 0 +} -echo "unknown" +check_ct_pid_1() { + if test -r /run/systemd/container + then + translate_container_name "$(head -n1 /run/systemd/container)" \ + && return 0 + fi + + if test -r /proc/1/environ + then + translate_container_name "$( + LC_ALL=C tr '\000' '\n' /dev/null + then + # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + echo wsl + elif test -d /var/.cagefs + then + # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs + # CageFS is not "really" a container, but it isn't a chroot either. + echo cagefs + elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status + then + # Linux-VServer + if grep -q -x -F 'VxID: 0' /proc/self/status + then + # host + return 1 + else + # guest + echo linux_vserver + fi + else + return 1 + fi +} + +check_ct_os_specific() ( + if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 + then + # FreeBSD jail + echo jail + return 0 + fi + + if is_command zonename && test "$(zonename)" != global + then + # Solaris zone + echo zone + return 0 + fi + + return 1 +) + + +# Check for hypervisor + +guess_hypervisor_from_cpu_model() { + case $1 + in + (*\ KVM\ *) + echo kvm ;; + (*\ QEMU\ *|QEMU\ *) + echo qemu ;; + (*) + return 1 ;; + esac +} + +has_vm_cpuinfo() { has_cpuinfo; } + +check_vm_cpuinfo() { + if grep -q -F 'User Mode Linux' /proc/cpuinfo \ + || grep -q -F 'UML' /proc/cpuinfo + then + # User Mode Linux + echo uml + elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo + then + # IBM PowerVM Lx86 (Linux/x86 emulator) + echo powervm_lx86 + elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo + then + # IBM SystemZ (S/390) + if test -f /proc/sysinfo + then + if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo + then + echo kvm-s390 + return 0 + elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo + then + echo zvm + return 0 + elif grep -q -e '^LPAR ' /proc/sysinfo + then + echo zvm-lpar + return 0 + fi + fi + return 1 + else + if grep -q -e '^model name.*:' /proc/cpuinfo + then + sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ + | while read -r _cpu_model + do + guess_hypervisor_from_cpu_model "${_cpu_model}" + done \ + | sort \ + | uniq -c \ + | awk ' + { if ($1 > most_c) { most_c = $1; most_s = $2 } } + END { + if (most_s) print most_s + exit !most_s + }' \ + && return 0 + fi + return 1 + fi +} + +check_vm_arch_specific() { + case ${arch} + in + (ppc64|ppc64le) + # Check PPC64 LPAR, KVM + + # example /proc/cpuinfo line indicating 'not baremetal' + # platform : pSeries + # + # example /proc/ppc64/lparcfg systemtype line + # system_type=IBM pSeries (emulated by qemu) + + if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo + then + if test -e /proc/ppc64/lparcfg + then + # Assume LPAR, now detect shared or dedicated + if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg + then + echo powervm-shared + return 0 + else + echo powervm-dedicated + return 0 + fi + fi + fi + ;; + (sparc*) + # Check for SPARC LDoms + + if test -e /dev/mdesc + then + if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp + then + # control LDom + return 1 + else + # guest LDom + echo ldom-sparc + fi + + # MDPROP=/usr/lib/ldoms/mdprop.py + # if test -x "${MDPROP}" + # then + # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" + # then + # echo ldoms-root + # echo ldoms-io + # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" + # then + # echo ldoms-io + # fi + # fi + return 0 + fi + ;; + (i?86|x86*|amd64|i86pc) + # Check CPUID + # + # Many fullvirt hypervisors give an indication through CPUID. Use + # the virt-what helper program to get this information if available. + + for CPUID_HELPER in \ + $(command -v virt-what-cpuid-helper 2>/dev/null) \ + /usr/lib/x86_64-*/virt-what-cpuid-helper \ + /usr/lib/i?86-*/virt-what-cpuid-helper \ + /usr/lib/virt-what/virt-what-cpuid-helper + do + if test -x "${CPUID_HELPER:?}"; then break; fi + done + + if test -x "${CPUID_HELPER-}" + then + case $(command "${CPUID_HELPER}") + in + ('bhyve bhyve ') + echo bhyve + ;; + ('LKVMLKVMLKVM') + echo lkvm + ;; + ('KVMKVMKVM') + echo kvm + ;; + ('TCGTCGTCGTCG') + echo qemu-tcg + ;; + ('Microsoft Hv') + # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx + echo hyperv + ;; + ('OpenBSDVMM58') + # OpenBSD/VMM + echo openbsd_vmm + ;; + ('VMwareVMware') + # check added by Chetan Loke. + echo vmware + ;; + ('XenVMMXenVMM') + if has dmi + then + # https://access.redhat.com/solutions/222903 + echo xen-hvm + else + echo xen-paravirt + fi + ;; + (*) + return 1 ;; + esac + return 0 + fi + + unset CPUID_HELPER + + # VMM CPUID flag denotes that this system is running under a VMM + if is_oneof "${uname_s}" Darwin + then + get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ + && return 0 + fi + if has_cpuinfo \ + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + then + return 0 + fi + ;; + (ia64) + if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend + then + # PV-on-HVM drivers installed in a Xen guest + echo xen-hvm + return 0 + fi + ;; + esac + return 1 +} + +has_vm_dmi() { + # Check for various products in SMBIOS/DMI. + # Note that DMI doesn't exist on all architectures (only x86 and some ARM). + # On other architectures the $dmi variable will be empty. + + if test -d /sys/class/dmi/id/ + then + dmi_sysfs=/sys/class/dmi/id + elif test -d /sys/devices/virtual/dmi/id/ + then + dmi_sysfs=/sys/devices/virtual/dmi/id + fi + + # shellcheck disable=SC2015 + { + is_command dmidecode \ + && ( + # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line + dmi_out=$(dmidecode 2>&1) \ + && ! printf '%s\n' "${dmi_out}" \ + | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' + ) \ + || test -d "${dmi_sysfs}" + } +} + +check_vm_dmi() { + case $(get_dmi_field system-product-name) + in + (*.metal) + if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' + then + # AWS EC2 bare metal -> no virtualisation + return 1 + fi + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('Google Compute Engine') + echo gce + return 0 + ;; + ('RHEV Hypervisor') + # Red Hat Enterprise Virtualization + echo rhev + return 0 + ;; + ('KVM'|'Bochs'|'KVM Virtual Machine') + echo kvm + return 0 + ;; + ('Parallels Virtual Platform') + echo parallels + return 0 + ;; + ('VirtualBox') + echo virtualbox + return 0 + ;; + ('VMware Virtual Platform') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field system-manufacturer) + in + ('Alibaba'*) + case $(get_dmi_field system-product-name) + in + ('Alibaba Cloud ECS') + echo alibaba-ecs + ;; + (*) + echo alibaba + ;; + esac + return 0 + ;; + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('innotek GmbH'|'Oracle Corporation') + echo virtualbox + return 0 + ;; + ('Joyent') + if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' + then + # SmartOS KVM + echo kvm-smartdc_hvm + return 0 + fi + ;; + ('Microsoft Corporation'*) + if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' + then + if test -e /proc/irq/7/hyperv \ + || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null + then + echo hyperv + return 0 + fi + + case $(get_dmi_field system-version) + in + (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) + echo virtualpc + return 0 + ;; + (*) + echo hyperv + return 0 + ;; + esac + fi + ;; + ('Nutanix') + # Nutanix AHV. Similar to KVM. + if test "$(get_dmi_field system-product-name)" = 'AHV' + then + echo nutanix_ahv + return 0 + fi + ;; + ('oVirt') + echo ovirt + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('QEMU') + echo qemu + return 0 + ;; + ('VMware, Inc.') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field bios-vendor) + in + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('innotek GmbH') + echo virtualbox + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('Xen') + if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' + then + # AWS on Xen + echo aws-xen + return 0 + fi + ;; + esac + + return 1 +} + +check_vm_hyp_specific() { + if is_command vmware-checkvm && vmware-checkvm >/dev/null + then + # vmware-checkvm is provided by VMware's open-vm-tools + echo vmware + return 0 + elif test -d /proc/xen + then + test -r /proc/xen/capabilities && + if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null + then + # Xen dom0 + return 1 + else + # Xen domU + echo xen + return 0 + fi + fi + return 1 +} + +has_vm_dt() { + # OpenFirmware/Das U-Boot device-tree + test -d /proc/device-tree +} + +check_vm_dt() { + case ${arch} + in + (arm|aarch64) + if test -r /proc/device-tree/hypervisor/compatible + then + if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible + then + echo xen + return 0 + elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible + then + # e.g. VMware ESXi on ARM + echo vmware + return 0 + fi + fi + if glob_exists /proc/device-tree/fw-cfg@*/compatible + then + # qemu,fw-cfg-mmio + sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 + return 0 + fi + if grep -q -F 'dummy-virt' /proc/device-tree/compatible + then + echo lkvm + return 0 + fi + ;; + (ppc64*) + if test -d /proc/device-tree/hypervisor \ + && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible + then + # We are running as a spapr KVM guest on ppc64 + echo kvm-spapr + return 0 + fi + if test -r /proc/device-tree/ibm,partition-name \ + && test -r /proc/device-tree/hmc-managed\? \ + && test -r /proc/device-tree/chosen/qemu,graphic-width + then + echo powervm + fi + ;; + esac + return 1 +} + +has_vm_sys_hypervisor() { + test -d /sys/hypervisor/ +} + +check_vm_sys_hypervisor() { + test -r /sys/hypervisor/type && + case $(head -n1 /sys/hypervisor/type) + in + (xen) + # Ordinary kernel with pv_ops. There does not seem to be + # enough information at present to tell whether this is dom0 + # or domU. + echo xen + return 0 + ;; + esac + return 1 +} + +check_vm_os_specific() { + _hyp_generic=false + + case ${uname_s} + in + (Darwin) + if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ + && test "${hv_vmm_present}" -ne 0 + then + _hyp_generic=true + fi + ;; + (FreeBSD) + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest=$(get_sysctl kern.vm_guest | tolower) && + case ${vm_guest} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${vm_guest} + in + (hv) echo hyperv ;; + (vbox) echo virtualbox ;; + (*) echo "${vm_guest}" ;; + esac + return 0 + ;; + esac + ;; + (NetBSD) + machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && + case ${machdep_hv} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${machdep_hv} + in + (hyper-v) echo hyperv ;; + (xenhvm*) echo xen-hvm ;; + (xenpv*) echo xen-pv ;; + (xen*) echo xen ;; + (*) echo "${machdep_hv}" ;; + esac + return 0 + ;; + esac + ;; + (OpenBSD) + if is_command hostctl && glob_exists /dev/pvbus[0-9]* + then + for _pvbus in /dev/pvbus[0-9]* + do + _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue + case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') + in + (KVM) echo kvm ;; + (Hyper-V) echo hyperv ;; + (VMware) echo vmware ;; + (Xen) echo xen ;; + (bhyve) echo bhyve ;; + (OpenBSD) echo openbsd_vmm ;; + esac + return 0 + done + fi + ;; + (SunOS) + diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') + # NOTE: Don't use -e or -F in Solaris grep + if printf '%s\n' "${diag_conf}" | grep -q -i QEMU + then + echo qemu + return 0 + elif printf '%s\n' "${diag_conf}" | grep -q -i VMware + then + echo vmware + return 0 + fi + ;; + (Linux) + if is_command dmesg + then + while read -r line + do + case ${line} + in + ('Booting paravirtualized kernel on ') + case $(expr "${line}" : '.* kernel on \(.*\)') + in + ('Xen') + echo xen-pv; return 0 ;; + ('bare hardware') + return 1 ;; + esac + ;; + ('Hypervisor detected') + case $(expr "${line}" : '.*: *\(.*\)') + in + ('ACRN') + echo acrn ;; + ('Jailhouse') + echo jailhouse ;; + ('KVM') + echo kvm ;; + ('Microsoft Hyper-V') + echo hyperv ;; + ('VMware') + echo vmware ;; + ('Xen HVM') + echo xen-hvm ;; + ('Xen PV') + echo xen-pv ;; + esac + return 0 + ;; + (lpar:*' under hypervisor') + return 0 ;; + esac + done <<-EOF + $(dmesg 2>/dev/null | awk ' + /Booting paravirtualized kernel on / + /Hypervisor detected: / + /lpar: .* under hypervisor/ + ') + EOF + fi + esac + + # Try to guess hypervisor based on CPU model (sysctl hw.model if available) + if cpu_model=$(get_sysctl hw.model) + then + guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 + fi + + if ${_hyp_generic} + then + # cannot say which hypervisor, but one was detected + return 0 + else + return 1 + fi +} + +run_stage() { + if type "has_$1_$2" >/dev/null 2>&1 + then + "has_$1_$2" + else + true + fi \ + && "check_$1_$2" +} + + +# Execute container stages + +for stage in \ + pid_1 cgroup files os_specific +do + ctengine=$(run_stage ct ${stage}) || continue + is_contained=true + if test -n "${ctengine}" + then + echo container on "${ctengine}" + break + fi +done +if ${is_contained:-false} && test -z "${ctengine}" +then + # none of the stages could determine the specific container engine, but + # we are running in some container. + echo container +fi + + +# Execute virtual machine / hypervisor stages + +for stage in \ + os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific +do + hypervisor=$(run_stage vm ${stage}) || continue + is_virtual=true + if test -n "${hypervisor}" + then + echo virtual by "${hypervisor}" + break + fi +done +if ${is_virtual:-false} && test -z "${hypervisor}" +then + # none of the stages could determine the specific hypervisor, but + # we are virtual. + echo virtual +fi From abc6d009b21b0d1ce3fc5107201e30740f127200 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:29:41 +0200 Subject: [PATCH 05/77] [explorer/machine_type] Print top most machine layer as first line (fallback to physical) --- cdist/conf/explorer/machine_type | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 29f98849..90c441da 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -26,17 +26,19 @@ # VERB does not have a special meaning, it is just for better readability. # # e.g. +# container # container on lxc # virtual by kvm-spapr # -# The third word of each line can be composed of different parts concatenated with a `-' -# (minus) character, with each component being a specification of the previous, -# e.g.: +# The third word of each line (except the first) can be composed of different +# parts concatenated with a `-' (minus) character, with each component being +# a specification of the previous, e.g.: # - lxc-libvirt (LXC container, managed by libvirt) # - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) # - xen-hvm / xen-pv (Xen HVM vs para-virtualization) # -# If this explorer cannot determine any information it will print nothing. +# If this explorer cannot collect enough information about virtualization it +# will fall back to 'physical'. # # Add /sbin and /usr/sbin to the path so we can find system @@ -121,6 +123,10 @@ get_sysctl() { is_command sysctl && sysctl -n "$1" 2>/dev/null } +detected_layer() { + test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" +} + # Check for container @@ -895,6 +901,7 @@ for stage in \ pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue + detected_layer 'container' is_contained=true if test -n "${ctengine}" then @@ -916,6 +923,7 @@ for stage in \ os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue + detected_layer 'virtual machine' is_virtual=true if test -n "${hypervisor}" then @@ -929,3 +937,8 @@ then # we are virtual. echo virtual fi + + +# Fallback + +detected_layer physical From 2ffa895f578f12d615ed65b9a93f3e2ae07f1d07 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 19:57:24 +0200 Subject: [PATCH 06/77] [explorer/machine_type] Remove CPUID check it's a lot of code and depends on a binary helper unlikely to be installed. --- cdist/conf/explorer/machine_type | 59 -------------------------------- 1 file changed, 59 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 90c441da..7ce035e3 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -379,65 +379,6 @@ check_vm_arch_specific() { fi ;; (i?86|x86*|amd64|i86pc) - # Check CPUID - # - # Many fullvirt hypervisors give an indication through CPUID. Use - # the virt-what helper program to get this information if available. - - for CPUID_HELPER in \ - $(command -v virt-what-cpuid-helper 2>/dev/null) \ - /usr/lib/x86_64-*/virt-what-cpuid-helper \ - /usr/lib/i?86-*/virt-what-cpuid-helper \ - /usr/lib/virt-what/virt-what-cpuid-helper - do - if test -x "${CPUID_HELPER:?}"; then break; fi - done - - if test -x "${CPUID_HELPER-}" - then - case $(command "${CPUID_HELPER}") - in - ('bhyve bhyve ') - echo bhyve - ;; - ('LKVMLKVMLKVM') - echo lkvm - ;; - ('KVMKVMKVM') - echo kvm - ;; - ('TCGTCGTCGTCG') - echo qemu-tcg - ;; - ('Microsoft Hv') - # http://blogs.msdn.com/b/sqlosteam/archive/2010/10/30/is-this-real-the-metaphysics-of-hardware-virtualization.aspx - echo hyperv - ;; - ('OpenBSDVMM58') - # OpenBSD/VMM - echo openbsd_vmm - ;; - ('VMwareVMware') - # check added by Chetan Loke. - echo vmware - ;; - ('XenVMMXenVMM') - if has dmi - then - # https://access.redhat.com/solutions/222903 - echo xen-hvm - else - echo xen-paravirt - fi - ;; - (*) - return 1 ;; - esac - return 0 - fi - - unset CPUID_HELPER - # VMM CPUID flag denotes that this system is running under a VMM if is_oneof "${uname_s}" Darwin then From 23fbfaf0352eadaabd43504d3ee074bb9f696fcd Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 21:29:24 +0200 Subject: [PATCH 07/77] [explorer/machine_type] Use systemd-detect-virt (if available) to detect containers and VMs --- cdist/conf/explorer/machine_type | 49 ++++++++++++++++++++++++++++++-- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 7ce035e3..1a38fda0 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -130,6 +130,25 @@ detected_layer() { # Check for container +has_ct_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -c' +} + +check_ct_systemd() ( + _ctengine=$(systemd-detect-virt -c 2>/dev/null) && + case ${_ctengine} + in + (''|'none') + return 1 ;; + ('container-other') + return 0 ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + echo "${_ctengine}" ;; + esac +) + has_ct_pid_1() { test -r /run/systemd/container -o -r /proc/1/environ } @@ -267,6 +286,32 @@ guess_hypervisor_from_cpu_model() { esac } +has_vm_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' +} + +check_vm_systemd() ( + _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && + case ${_hypervisor} + in + (''|'none') + return 1 ;; + ('amazon') + echo aws ;; + ('bochs') + echo kvm ;; + ('microsoft') + # assumption + echo hyperv ;; + ('oracle') + echo virtualbox ;; + ('vm-other') + return 0 ;; + (*) + echo "${_hypervisor}" ;; + esac +) + has_vm_cpuinfo() { has_cpuinfo; } check_vm_cpuinfo() { @@ -839,7 +884,7 @@ run_stage() { # Execute container stages for stage in \ - pid_1 cgroup files os_specific + systemd pid_1 cgroup files os_specific do ctengine=$(run_stage ct ${stage}) || continue detected_layer 'container' @@ -861,7 +906,7 @@ fi # Execute virtual machine / hypervisor stages for stage in \ - os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific + systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific do hypervisor=$(run_stage vm ${stage}) || continue detected_layer 'virtual machine' From 4a05669765c2c00c19af0ef1b607b0f2efa10f42 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 31 Jul 2021 22:01:28 +0200 Subject: [PATCH 08/77] [explorer/machine_type] Implement chroot detection --- cdist/conf/explorer/machine_type | 53 ++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1a38fda0..10c914ba 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -53,6 +53,21 @@ uname_s=$(uname -s) is_command() { command -v "$1" >/dev/null 2>&1; } +files_same() { + # shellcheck disable=SC2012 + LC_ALL=C df -P "$1" "$2" 2>/dev/null | { + read -r _ # skip header line + read -r fs1 _ _ _ _ mp1 + read -r fs2 _ _ _ _ mp2 + test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 + } && + ls -1Ldi "$1" "$2" 2>/dev/null | { + read -r ino1 _ + read -r ino2 _ + test "${ino1}" = "${ino2}" || return 1 + } +} + is_oneof() ( x=$1; shift for y @@ -128,6 +143,32 @@ detected_layer() { } +# Check for chroot + +has_chroot_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' +} + +check_chroot_systemd() { + systemd-detect-virt -r +} + +has_chroot_debian_ischroot() { + is_command ischroot +} + +check_chroot_debian_ischroot() { + ischroot --default-false +} + +has_chroot_procfs() { + test -d /proc/ +} + +check_chroot_procfs() { + test -e /proc/1/root && ! files_same /proc/1/root / +} + # Check for container has_ct_systemd() { @@ -881,6 +922,18 @@ run_stage() { } +# Execute chroot stages + +for stage in \ + procfs debian_ischroot systemd +do + run_stage chroot ${stage} || continue + detected_layer 'chroot' + echo chroot + break +done + + # Execute container stages for stage in \ From 5af1317c2969995871da1d17b951de60dbc3fd09 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 16:40:20 +0200 Subject: [PATCH 09/77] [explorer/machine_type] Try to detect chroot path --- cdist/conf/explorer/machine_type | 38 ++++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 4 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 10c914ba..fa68ec4d 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -166,7 +166,28 @@ has_chroot_procfs() { } check_chroot_procfs() { - test -e /proc/1/root && ! files_same /proc/1/root / + if test -e /proc/1/root && ! files_same /proc/1/root / + then + # try to determine where the chroot has been mounted + ( + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # Get mount point + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + fi + ) + return 0 + fi + return 1 } # Check for container @@ -927,11 +948,20 @@ run_stage() { for stage in \ procfs debian_ischroot systemd do - run_stage chroot ${stage} || continue + chrootpnt=$(run_stage chroot ${stage}) || continue + is_chrooted=true detected_layer 'chroot' - echo chroot - break + if test -n "${chrootpnt}" + then + echo chroot at "${chrootpnt}" + break + fi done +if ${is_chrooted:-false} && test -z "${chrootpnt}" +then + # could determine chroot, but not its mount point + echo chroot +fi # Execute container stages From 05c2a62191f533ceff1e074df571278a103d75ab Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 1 Aug 2021 23:09:02 +0200 Subject: [PATCH 10/77] [explorer/machine_type] Implement chroot detection using /proc/.../mountinfo --- cdist/conf/explorer/machine_type | 64 +++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index fa68ec4d..00646c75 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -165,30 +165,52 @@ has_chroot_procfs() { test -d /proc/ } -check_chroot_procfs() { +check_chroot_procfs() ( + is_chroot=false # default if test -e /proc/1/root && ! files_same /proc/1/root / then - # try to determine where the chroot has been mounted - ( - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # Get mount point - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - fi - ) - return 0 + is_chroot=true fi - return 1 -} + if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo + then + has_mountinfo=true + cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true + fi + + if ${is_chroot} + then + # try to determine where the chroot has been mounted + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # escape chroot and find mount point by inode + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + elif ${has_mountinfo} + then + while read -r mntid _ _ _ cmntpnt _ + do + read -r _ _ _ _ hmntpnt _ <<-EOF + $(grep -e "^$((mntid)) " /proc/1/mountinfo) + EOF + printf '%s\n' "${hmntpnt%${cmntpnt}}" + done Date: Tue, 24 Aug 2021 08:09:47 +0200 Subject: [PATCH 11/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f9409d7e..8507c663 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,7 @@ next: * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) + * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) 6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) From 0546283d0ebc0058dbff68cec9d18b9281c26edd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:32:44 +0200 Subject: [PATCH 12/77] Update shellcheck disable --- cdist/conf/type/__apt_backports/manifest | 1 + cdist/conf/type/__package_pkg_freebsd/gencode-remote | 1 + cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 3 files changed, 3 insertions(+) diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index bc47d8de..6fcd9212 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,6 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 3f88f6bc..ca9aa45a 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,6 +37,7 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 + # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index cec25746..9694a64b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,6 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1090 +# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From 44741e714b16f7a00bf84bd54ae33eacc7593192 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:25:49 +0200 Subject: [PATCH 13/77] Release 6.9.8 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 8507c663..dcdc4b3d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) From b8eb6e984c1638e8e167394c6b3aa482cb8aad49 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 24 Aug 2021 20:47:50 +0200 Subject: [PATCH 14/77] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index dcdc4b3d..693d028f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Explorer machine_type: Rewrite (Dennis Camera) + 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) * New type: __apt_pin (Daniel Fancsali) From d7fdc8006f747be86b64a2790266b50219bab507 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 21:54:45 +0300 Subject: [PATCH 15/77] allow empty file --- cdist/conf/type/__sed/explorer/file | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index a40f71b3..5f0f4edd 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,13 +9,7 @@ fi if [ -f "$file" ] then - if [ -s "$file" ] - then - cat "$file" - else - echo "$file is empty" >&2 - exit 1 - fi + cat "$file" else echo "$file do not exist" >&2 exit 1 From 0f6e48dbc657a9d9145b965eacf79f021a1419f5 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:24:26 +0300 Subject: [PATCH 16/77] use $__object/tempfile in target instead of mktemp, add comments --- cdist/conf/type/__sed/gencode-remote | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 836506f0..df929b42 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -35,8 +35,11 @@ then exit 0 fi +# we can't use -i, because it's not posix, so we fly with tempfile and cp +# and we use cp because we want to preserve destination file's attributes + # shellcheck disable=SC2016 -echo 'tmp="$( mktemp )"' +echo 'tmp="$__object/tempfile"' echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" From 90488fcebcc00f3b4ea260867f8b31625df61e46 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:27:42 +0300 Subject: [PATCH 17/77] use -e --- cdist/conf/type/__sed/explorer/file | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 5f0f4edd..9262b613 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -7,10 +7,10 @@ else file="/$__object_id" fi -if [ -f "$file" ] +if [ ! -e "$file" ] then - cat "$file" -else echo "$file do not exist" >&2 exit 1 fi + +cat "$file" From b7f392fa371ff78591abb910f7e224e7587c81be Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:38:55 +0300 Subject: [PATCH 18/77] use -E for better compat (not really sure if it is posix at all) --- cdist/conf/type/__sed/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index df929b42..625448e6 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -24,7 +24,7 @@ sed_cmd='sed' if [ -f "$__object/parameter/regexp-extended" ] then - sed_cmd="$sed_cmd --regexp-extended" + sed_cmd="$sed_cmd -E" fi if echo "$script" \ From aabef7f44a98ad144a7e1fe68bfb562079916ef8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:40:06 +0300 Subject: [PATCH 19/77] remove reading script from file --- cdist/conf/type/__sed/gencode-remote | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index 625448e6..be7690e1 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -12,10 +12,6 @@ script="$( cat "$__object/parameter/script" )" if [ "$script" = '-' ] then script="$( cat "$__object/stdin" )" -elif - [ -f "$script" ] -then - script="$( cat "$script" )" fi file_from_target="$__object/explorer/file" From 5bf0c71e7af49ff6e7c1cccec192f763a019503e Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 14 Sep 2021 22:45:36 +0300 Subject: [PATCH 20/77] update man --- cdist/conf/type/__sed/man.rst | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst index 4a728184..86789363 100644 --- a/cdist/conf/type/__sed/man.rst +++ b/cdist/conf/type/__sed/man.rst @@ -8,35 +8,40 @@ cdist-type__sed - Transform text files with ``sed`` DESCRIPTION ----------- -TODO +Transform text files with ``sed``. REQUIRED MULTIPLE PARAMETERS ---------------------------- script - TODO + ``sed`` script. + If ``-`` then the script is read from ``stdin``. OPTIONAL PARAMETERS ------------------- file - TODO + Path to the file. Defaults to ``$__object_id``. onchange - TODO + Execute this command if ``sed`` changes file. BOOLEAN PARAMETERS ------------------ regexp-extended - TODO + Use extended regular expressions in the script. + Might not be supported with every ``sed`` version. EXAMPLES -------- + .. code-block:: sh - true + __sed /tmp/foobar --script 's/foo/bar/' + + echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - AUTHORS From cd4acde67ec9c3967e6e08450734cf65040e787f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 15 Sep 2021 09:22:27 +0300 Subject: [PATCH 21/77] grammar --- cdist/conf/type/__sed/explorer/file | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file index 9262b613..ec3d0fe8 100755 --- a/cdist/conf/type/__sed/explorer/file +++ b/cdist/conf/type/__sed/explorer/file @@ -9,7 +9,7 @@ fi if [ ! -e "$file" ] then - echo "$file do not exist" >&2 + echo "$file does not exist" >&2 exit 1 fi From 7b6789ddeb74450f70991efe957e3eb0f11043f6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:04:12 +0200 Subject: [PATCH 22/77] __package_update_index: fix complain about suite change 2 of 4th fix for ticket #861 --- .../type/__package_update_index/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index 803468b5..a10c16d3 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,7 +41,19 @@ fi case "$type" in yum) ;; apt) - echo "apt-get --quiet update" + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + + echo "apt-get --quiet $apt_opts update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) From 12787ffe2c48f5c6d534d10f8faadde30db477ac Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:12:20 +0200 Subject: [PATCH 23/77] __apt_source: fix complain about suite change 3 of 4th fix for ticket #861 --- cdist/conf/type/__apt_source/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 1e8592c6..973b0f6c 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,7 +22,21 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + +# run 'apt-get update' only if something changed with our sources.list file +# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get update || apt-get update\n' + printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" fi From d246e067109dc8b85ea89668f742affbb221d1a6 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 14:49:23 +0200 Subject: [PATCH 24/77] __apt_update_index: fix complain about suite change 1 of 4th fix for ticket #861 --- .../conf/type/__apt_update_index/gencode-remote | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 70b59710..2d7f9030 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,9 +18,23 @@ # along with cdist. If not, see . # + +# There are special arguments to apt(8) to prevent aborts if apt woudn't been +# updated after the 19th April 2021 till the bullseye release. The additional +# arguments acknoledge the happend suite change (the apt(8) update does the +# same by itself). +# +# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter +# allows backward compatablility to pre-buster Debian versions. +# +# See more: ticket #861 +# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 +apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists +# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get update || apt-get update + apt-get $apt_opts update || apt-get $apt_opts update fi DONE From 3d7b31cbb43067271c2c510281f90f19dc5ec5fb Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Wed, 15 Sep 2021 15:22:16 +0200 Subject: [PATCH 25/77] __package_apt: fix complain about suite change the last fix for ticket #861 :-) --- cdist/conf/type/__package_apt/gencode-remote | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index fbfca330..79c0d9d3 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,12 +81,24 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) + # There are special arguments to apt(8) to prevent aborts if apt woudn't been + # updated after the 19th April 2021 till the bullseye release. The additional + # arguments acknoledge the happend suite change (the apt(8) update does the + # same by itself). + # + # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter + # allows backward compatablility to pre-buster Debian versions. + # + # See more: ticket #861 + # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 + apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" + # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get update > /dev/null 2>&1 || true +then echo apt-get $apt_opts update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then From 72ff48154c958ebdedc8b6d919b6f372efa63be2 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 16 Sep 2021 21:36:39 +0300 Subject: [PATCH 26/77] add comments, add -u to diff --- cdist/conf/type/__sed/gencode-remote | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote index be7690e1..f99c5a88 100755 --- a/cdist/conf/type/__sed/gencode-remote +++ b/cdist/conf/type/__sed/gencode-remote @@ -14,6 +14,8 @@ then script="$( cat "$__object/stdin" )" fi +# since stdin is not available in explorer, we pull file from target with explorer + file_from_target="$__object/explorer/file" sed_cmd='sed' @@ -23,10 +25,10 @@ then sed_cmd="$sed_cmd -E" fi -if echo "$script" \ - | "$sed_cmd" -f - "$file_from_target" \ - | diff "$file_from_target" - \ - > /dev/null +# do sed dry run, diff result and if no change, then there's nothing to do +# also redirect diff's output to stderr for debugging purposes + +if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 then exit 0 fi From bf222d05435000063930ff199bd0dec94d7d8cd5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 21 Sep 2021 08:55:54 +0200 Subject: [PATCH 27/77] ++changelog --- docs/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog b/docs/changelog index 693d028f..ab637f5f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,11 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) + * New type: __sed (Ander Punnar) + * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __apt_source: Fix complaint about suite change (Matthias Stecher) + * Type __package_apt: Fix complaint about suite change (Matthias Stecher) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 15c642a9b7af42c368eb5a8e22cab5f07b39b95e Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:06:45 +0200 Subject: [PATCH 28/77] [__debconf_set_selections] Fix --file not being supported Even if deprecated, the parameter *must* be supported, which isn't the case right now. This was due to a misunderstanding of how deprecating parameters work, see: https://www.cdi.st/manual/latest/cdist-type.html#deprecated-parameters --- .../conf/type/__debconf_set_selections/parameter/deprecated/file | 1 + .../__debconf_set_selections/parameter/{deprecated => optional} | 0 2 files changed, 1 insertion(+) create mode 100644 cdist/conf/type/__debconf_set_selections/parameter/deprecated/file rename cdist/conf/type/__debconf_set_selections/parameter/{deprecated => optional} (100%) diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file new file mode 100644 index 00000000..09db545a --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file @@ -0,0 +1 @@ +'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated b/cdist/conf/type/__debconf_set_selections/parameter/optional similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/deprecated rename to cdist/conf/type/__debconf_set_selections/parameter/optional From 5b7cca99f73e4ec3dd5cdb90e4dbfd40ec8fe349 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 12:09:42 +0200 Subject: [PATCH 29/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index ab637f5f..142602c8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) + * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From fc9bd40c9a34cf54e2d6cfd2fec089bd25e1b172 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:14:57 +0200 Subject: [PATCH 30/77] Improve bullseye support, perticularly __letsencrypt_cert --- cdist/conf/type/__grafana_dashboard/manifest | 2 +- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index d145c4c3..0d944482 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*) + 10*|11*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 6394f629..638a99e0 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*) + 10*|11*) __package_apt certbot ;; From 560374a6861281695b61c8c2298123be41d92326 Mon Sep 17 00:00:00 2001 From: Evilham Date: Fri, 1 Oct 2021 13:16:11 +0200 Subject: [PATCH 31/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 142602c8..6f717cf4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) + * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From c33d99ee120ad180daba144be80a25d77b473f56 Mon Sep 17 00:00:00 2001 From: Evilham Date: Sun, 31 Oct 2021 17:38:10 +0100 Subject: [PATCH 32/77] [__haproxy_dualstack] New type with PROXY protocol support This is backwards compatible with what is already used internally @ungleich, but adds on top of that the ability to customise ports and, most importantly, it adds PROXY protocol support. --- .../conf/type/__haproxy_dualstack/files/http | 8 + .../conf/type/__haproxy_dualstack/files/https | 10 ++ .../conf/type/__haproxy_dualstack/files/imaps | 12 ++ .../conf/type/__haproxy_dualstack/files/smtps | 12 ++ cdist/conf/type/__haproxy_dualstack/man.rst | 121 ++++++++++++++ cdist/conf/type/__haproxy_dualstack/manifest | 155 ++++++++++++++++++ .../parameter/default/protocol | 1 + .../parameter/optional_multiple | 3 + cdist/conf/type/__haproxy_dualstack/singleton | 0 9 files changed, 322 insertions(+) create mode 100644 cdist/conf/type/__haproxy_dualstack/files/http create mode 100644 cdist/conf/type/__haproxy_dualstack/files/https create mode 100644 cdist/conf/type/__haproxy_dualstack/files/imaps create mode 100644 cdist/conf/type/__haproxy_dualstack/files/smtps create mode 100644 cdist/conf/type/__haproxy_dualstack/man.rst create mode 100644 cdist/conf/type/__haproxy_dualstack/manifest create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/default/protocol create mode 100644 cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple create mode 100644 cdist/conf/type/__haproxy_dualstack/singleton diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http new file mode 100644 index 00000000..0508a465 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/http @@ -0,0 +1,8 @@ +frontend http + bind BIND@:80 + mode http + option httplog + default_backend http + +backend http + mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https new file mode 100644 index 00000000..73deac46 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/https @@ -0,0 +1,10 @@ +frontend https + bind BIND@:443 + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend https + +backend https + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps new file mode 100644 index 00000000..b1ec3793 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/imaps @@ -0,0 +1,12 @@ +frontend imaps + bind BIND@:143 + bind BIND@:993 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend imaps + +backend imaps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps new file mode 100644 index 00000000..dce6ed4a --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/files/smtps @@ -0,0 +1,12 @@ +frontend smtps + bind BIND@:25 + bind BIND@:465 + + mode tcp + option tcplog + tcp-request inspect-delay 5s + tcp-request content accept if { req_ssl_hello_type 1 } + default_backend smtps + +backend smtps + mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst new file mode 100644 index 00000000..6c131cbe --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/man.rst @@ -0,0 +1,121 @@ +cdist-type__haproxy_dualstack(7) +================================ + + +NAME +---- +cdist-type__haproxy_dualstack - Proxy services from a dual-stack server + + +DESCRIPTION +----------- +This (singleton) type installs and configures haproxy to act as a dual-stack +proxy for single-stack services. + +This can be useful to add IPv4 support to IPv6-only services while only using +one IPv4 for many such services. + +By default this type uses the plain TCP proxy mode, which means that there is no +need for TLS termination on this host when SNI is supported. +This also means that proxied services will not receive the client's IP address, +but will see the proxy's IP address instead (that of `$__target_host`). + +This can be solved by using the PROXY protocol, but do take into account that, +e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same +port, so you will need to use other ports for that. + +As a recommendation in this type: use TCP ports 8080 and 591 respectively to +serve HTTP and HTTPS using the PROXY protocol. + +See the EXAMPLES for more details. + + +OPTIONAL PARAMETERS +------------------- +v4proxy + Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. + In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is + the IP address actually providing the proxied services. + The full format of this argument is: + `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` + Where starting with `proxy:` determines that the PROXY protocol must be + used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT + override for the given PROTOCOL (see `--protocol`), if not present the + PROTOCOL's default port will be used. + + +v6proxy + Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. + In its simplest use, it must be a NAME with an `A` DNS entry, which is + the IP address actually providing the proxied services. + See `--v4proxy` for more options and details. + +protocol + Can be passed multiple times or as a space-separated list of protocols. + Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. + This defaults to: `http https imaps smtps`. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Proxy the IPv6-only services so IPv4-only clients can access them + # This uses HAProxy's TCP mode for http, https, imaps and smtps + __haproxy_dualstack \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them + # Note this means that the backend IPv6-only server will only see + # the IPv6 address of the haproxy host managed by cdist, which can be + # troublesome if this information is relevant for analytics/security/... + # See the PROXY example below + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy ipv6.chat \ + --v4proxy matrix.ungleich.ch + + # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling + # IPv4-only clients to access them while maintaining the client's IP address + __haproxy_dualstack \ + --protocol http --protocol https \ + --v4proxy proxy:ipv6.chat:http=8080:https=591 \ + --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 + # Note however that the PROXY protocol is not compatible with regular + # HTTP(S) protocols, so your nginx will have to listen on different ports + # with the PROXY settings. + # Note that you will need to restrict access to the 8080 port to prevent + # Client IP spoofing. + # This can be something like: + # server { + # # listen for regular HTTP connections + # listen [::]:80 default_server; + # listen 80 default_server; + # # listen for PROXY HTTP connections + # listen [::]:8080 proxy_protocol; + # # Accept the Client's IP from the PROXY protocol + # real_ip_header proxy_protocol; + # } + + +SEE ALSO +-------- +- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ +- https://www.haproxy.com/blog/haproxy/proxy-protocol/ +- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ + + +AUTHORS +------- +ungleich +Evilham + + +COPYING +------- +Copyright \(C) 2021 ungleich glarus ag. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest new file mode 100644 index 00000000..d110eea6 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/manifest @@ -0,0 +1,155 @@ +#!/bin/sh -eu + +__package haproxy +require="__package/haproxy" __start_on_boot haproxy + +tmpdir="$__object/files" +mkdir "$tmpdir" +configtmp="$__object/files/haproxy.cfg" + +os=$(cat "$__global/explorer/os") +case $os in + freebsd) + CONFIG_FILE="/usr/local/etc/haproxy.conf" + cat < "$configtmp" +global + maxconn 4000 + user nobody + group nogroup + daemon + +EOF + + ;; + *) + CONFIG_FILE="/etc/haproxy/haproxy.cfg" + cat < "$configtmp" +global + log [::1] local2 + chroot /var/lib/haproxy + pidfile /var/run/haproxy.pid + maxconn 4000 + user haproxy + group haproxy + daemon + + # turn on stats unix socket + stats socket /var/lib/haproxy/stats + +EOF + ;; +esac + +cat <> "$configtmp" +defaults + retries 3 + log global + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 1m + timeout server 1m + timeout http-keep-alive 10s + timeout check 10s +EOF + +dig_cmd="$(command -v dig || true)" +get_ip() { + # Usage: get_ip (ipv4|ipv6) NAME + # uses "dig" if available, else fallback to "host" + case $1 in + ipv4) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short A "$2" + else + host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' + fi + ;; + ipv6) + if [ -n "${dig_cmd}" ]; then + ${dig_cmd} +short AAAA "$2" + else + host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' + fi + ;; + esac +} + +PROTOCOLS="$(cat "$__object/parameter/protocol")" + +for proxy in v4proxy v6proxy; do + param=$__object/parameter/$proxy + # no backend? skip generating code + if [ ! -f "$param" ]; then + continue + fi + + # turn backend name into bind parameter: v4backend -> ipv4@ + bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') + + case $bind in + ipv4) + backendproto=ipv6 + ;; + ipv6) + backendproto=ipv4 + ;; + esac + + for proto in ${PROTOCOLS}; do + # Add protocol "header" + printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" + + sed -e "s/BIND/$bind/" \ + -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ + -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ + "$__type/files/$proto" >> "$configtmp" + + while read -r hostdefinition; do + if echo "$hostdefinition" | grep -qE '^proxy:'; then + # Proxy protocol was requested + host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" + send_proxy=" send-proxy" + else + # Just use tcp proxy mode + host="$hostdefinition" + send_proxy="" + fi + if echo "$hostdefinition" | grep -qE ":${proto}="; then + # Use custom port definition if requested + port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" + else + # Else use the default + port="" + fi + servername=$host + + res=$(get_ip "$bind" "$servername") + + if [ -z "$res" ]; then + echo "$servername does not resolve - aborting config" >&2 + exit 1 + fi + + # Treat protocols without TLS+SNI specially + if [ "$proto" = http ]; then + echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" + else + echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" + fi + + # Create the "server" itself. + # Note that port and send_proxy will be empty unless + # they were requested by the type user + echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" + + done < "$param" + done +done + +# Create config file +require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 + +require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ + --pattern "^__file${CONFIG_FILE}" \ + --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol new file mode 100644 index 00000000..dc8bb7bf --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol @@ -0,0 +1 @@ +http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple new file mode 100644 index 00000000..8c482bd4 --- /dev/null +++ b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple @@ -0,0 +1,3 @@ +protocol +v4proxy +v6proxy diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton new file mode 100644 index 00000000..e69de29b From e2500248f2ddc83129e77f2e6b8dffb64904dbae Mon Sep 17 00:00:00 2001 From: Evilham Date: Wed, 3 Nov 2021 11:03:33 +0100 Subject: [PATCH 33/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 6f717cf4..99a8c08b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) + * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) From 3a321469a8ba5aea55220bd70bd4900de732e917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 16 Nov 2021 11:11:45 +0100 Subject: [PATCH 34/77] Python 3.10: collections.X -> collections.abc.X --- cdist/integration.py | 2 +- cdist/util/fsproperty.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/integration.py b/cdist/integration.py index 17b65f09..04470ea7 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.Iterable): + elif isinstance(host, collections.abc.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 09e9cc19..6bf935e8 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.MutableSequence): +class FileList(collections.abc.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.MutableSequence): self.__write(lines) -class DirectoryDict(collections.MutableMapping): +class DirectoryDict(collections.abc.MutableMapping): """A dict that stores it's items as files in a directory. """ From 6e3ad11ea0177865e7e0288b1765267df8d5d020 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:07:28 +0100 Subject: [PATCH 35/77] [__package_upgrade_all] Add new --apt-with-new-pkgs argument --- cdist/conf/type/__package_upgrade_all/gencode-remote | 6 +++++- cdist/conf/type/__package_upgrade_all/man.rst | 8 ++++++++ cdist/conf/type/__package_upgrade_all/parameter/boolean | 1 + 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 38aa001e..d332e851 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,6 +28,10 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" +if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then + apt_with_new_pkgs="--with-new-pkgs" +fi + if [ -f "$type" ]; then type="$(cat "$type")" else @@ -54,7 +58,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget upgrade" + else echo "$aptget $apt_with_new_pkgs upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index e9e2b8ce..0c116bac 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,6 +33,14 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. +apt-with-new-pkg + Allow installing new packages when used in conjunction with + upgrade. This is useful if the update of an installed package + requires new dependencies to be installed. Instead of holding the + package back upgrade will upgrade the package and install the new + dependencies. Note that upgrade with this option will never remove + packages, only allow adding new ones. + apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index 7a56a34b..cd22eb90 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,2 +1,3 @@ apt-clean apt-dist-upgrade +apt-with-new-pkgs From c2c5668b704e1648ff6c8fb88219badddd028346 Mon Sep 17 00:00:00 2001 From: Evilham Date: Thu, 23 Dec 2021 20:08:49 +0100 Subject: [PATCH 36/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 99a8c08b..26d89057 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,7 @@ next: * New type: __haproxy_dualstack (Evilham and ungleich) * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) + * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) * Type __apt_source: Fix complaint about suite change (Matthias Stecher) * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) From 08ff41efded6e3112fc462ba13d9166e620b4082 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Tue, 8 Mar 2022 12:04:58 +0100 Subject: [PATCH 37/77] Added rm of tmpfile. --- cdist/conf/type/__ssh_authorized_key/gencode-remote | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 61c77fb9..cbffde94 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,6 +40,7 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" +rm -f "\$tmpfile" DONE } From e0150e779681e232f95bdbefd957c666f05daa89 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Mar 2022 16:16:44 +0100 Subject: [PATCH 38/77] ++changes --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 26d89057..81be51f6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -12,6 +12,8 @@ next: * Type __package_apt: Fix complaint about suite change (Matthias Stecher) * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) + * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) + * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From bd44c023d33eb51a09afad12b18c082f6a3ae36d Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 11 Jun 2021 11:22:31 +0100 Subject: [PATCH 39/77] Fix typos; add default priority; comments in generated files --- cdist/conf/type/__apt_pin/manifest | 5 +++++ cdist/conf/type/__apt_pin/parameter/default/priority | 1 + cdist/conf/type/__apt_pin/parameter/optional | 1 + cdist/conf/type/__apt_pin/parameter/required | 1 - 4 files changed, 7 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__apt_pin/parameter/default/priority diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index e72a8fdd..983b2b42 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -57,6 +57,11 @@ __file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF +# Created by cdist ${__type##*/} +# Do not change. Changes will be overwritten. +# + +# $name Package: $package Pin: $pin Pin-Priority: $priority diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority new file mode 100644 index 00000000..1b79f38e --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/priority @@ -0,0 +1 @@ +500 diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional index 52f01fd2..847e703d 100644 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -1,2 +1,3 @@ state package +priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required index 4b4e9741..c8572d92 100644 --- a/cdist/conf/type/__apt_pin/parameter/required +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -1,2 +1 @@ distribution -priority From 22039284f57f575defa0bc65c46b8bbcbe016cd8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sun, 10 Apr 2022 23:52:53 +0200 Subject: [PATCH 40/77] __file: make file uploading and attribute changes more atomic Fixes https://code.ungleich.ch/ungleich-public/cdist/pulls/331 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++++-------- cdist/conf/type/__file/gencode-remote | 14 +++++++++++++- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 231b6927..bea3d79c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -89,10 +89,26 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - tempfile_template="${destination}.cdist.XXXXXXXXXX" + upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + # Yes, we are aware that this is a race condition. + # However: + # a) cdist usually writes to directories that are not user writable + # (probably > 99.9%) + # b) if they are user owned, the user / attacker always wins + # (probably < 0.1%) + # c) the only case which we could improve are tmp directories and we + # don't think managing tmp directories with cdist is a typical case + # ("the rest %)" cat << DONE -destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +$__remote_exec $__target_host test -e $upload_destination && { + echo "Refusing to upload file to existing destination: $upload_destination" >&2 + exit 1 +} DONE + # Tell gencode-remote to where we uploaded the file so it can move + # it to its final destination. + echo "$upload_destination" > "$__object/files/upload-destination" + if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix @@ -103,12 +119,8 @@ DONE my_target_host="${__target_host}" fi cat << DONE -$__remote_copy "$source" "${my_target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE fi -# move uploaded file into place -cat << DONE -$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" -DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index f7a528fd..136520a7 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,6 +62,13 @@ set_mode() { case "$state_should" in present|exists) + if [ -f "$__object/files/upload-destination" ]; then + final_destination="$destination" + # We change the 'global' $destination variable here so we can + # change attributes of the new/uploaded file before moving it + # to it's final destination. + destination="$(cat "$__object/files/upload-destination")" + fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -81,6 +88,11 @@ case "$state_should" in fi fi done + if [ -f "$__object/files/upload-destination" ]; then + # move uploaded file into place + printf 'rm -rf "%s"\n' "$final_destination" + printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 From af54fe6febc186b0db6da00f2d06d5e87f4012f9 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 11 Apr 2022 00:04:41 +0200 Subject: [PATCH 41/77] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 81be51f6..3363be27 100644 --- a/docs/changelog +++ b/docs/changelog @@ -14,6 +14,7 @@ next: * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) + * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From cb0fa0f2e4d99891653b8871309e22f62ff393dd Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:39:29 +0100 Subject: [PATCH 42/77] force add-apt-repository to act in non-interactive mode --- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 84ebebfe..094366b9 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,7 +29,7 @@ fi case "$state_should" in present) - echo "add-apt-repository '$name'" + echo "add-apt-repository -y '$name'" ;; absent) echo "remove-apt-repository '$name'" From 54a5cb17b762a7fbb61225b36621c8a45e83536b Mon Sep 17 00:00:00 2001 From: Romain Dartigues Date: Fri, 24 Dec 2021 15:41:03 +0100 Subject: [PATCH 43/77] use add-apt-repository instead of add-apt-repository Remove `remove-apt-repository` which is now no longer needed; use `add-apt-repository` which allow removal through the `-r` flag. --- .../__apt_ppa/files/remove-apt-repository | 55 ------------------- cdist/conf/type/__apt_ppa/gencode-remote | 2 +- cdist/conf/type/__apt_ppa/manifest | 5 -- 3 files changed, 1 insertion(+), 61 deletions(-) delete mode 100755 cdist/conf/type/__apt_ppa/files/remove-apt-repository diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository deleted file mode 100755 index 3eb7d491..00000000 --- a/cdist/conf/type/__apt_ppa/files/remove-apt-repository +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env python -# -# Remove the given apt repository. -# -# Exit with: -# 0: if it worked -# 1: if not -# 2: on other error - -import os -import sys -from aptsources import distro, sourceslist -from softwareproperties import ppa -from softwareproperties.SoftwareProperties import SoftwareProperties - - -def remove_if_empty(file_name): - with open(file_name, 'r') as f: - if f.read().strip(): - return - os.unlink(file_name) - -def remove_repository(repository): - #print 'repository:', repository - codename = distro.get_distro().codename - #print 'codename:', codename - (line, file) = ppa.expand_ppa_line(repository.strip(), codename) - #print 'line:', line - #print 'file:', file - deb_source_entry = sourceslist.SourceEntry(line, file) - src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) - - try: - sp = SoftwareProperties() - sp.remove_source(deb_source_entry) - try: - # If there's a deb-src entry, remove that too - sp.remove_source(src_source_entry) - except: - pass - remove_if_empty(file) - return True - except ValueError: - print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line - return False - -if __name__ == '__main__': - if (len(sys.argv) != 2): - print >> sys.stderr, 'Error: need a repository as argument' - sys.exit(2) - repository = sys.argv[1] - if remove_repository(repository): - sys.exit(0) - else: - sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 094366b9..e41341b8 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -32,6 +32,6 @@ case "$state_should" in echo "add-apt-repository -y '$name'" ;; absent) - echo "remove-apt-repository '$name'" + echo "add-apt-repository -r -y '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index c6f4e876..57e85442 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,9 +20,4 @@ __package software-properties-common -require="__package/software-properties-common" \ - __file /usr/local/bin/remove-apt-repository \ - --source "$__type/files/remove-apt-repository" \ - --mode 0755 - require="$__object_name" __apt_update_index From 6f8c774cb08e29c8f0282a8a40a43f1d09c7248e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:16:10 +0200 Subject: [PATCH 44/77] workaround mktemp -u checking for write access Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index bea3d79c..42c9d1e7 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -89,7 +89,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then touch "$__object/files/set-attributes" # upload file to temp location - upload_destination="$(mktemp -u "${destination}.cdist.XXXXXXXXXX")" + upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 2df2578e36c688fcbdb32a87609bc263d2e7db9c Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:27:28 +0200 Subject: [PATCH 45/77] __file: remove the questionable check for uploadfile existence Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 42c9d1e7..aa7149c1 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -99,12 +99,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # c) the only case which we could improve are tmp directories and we # don't think managing tmp directories with cdist is a typical case # ("the rest %)" - cat << DONE -$__remote_exec $__target_host test -e $upload_destination && { - echo "Refusing to upload file to existing destination: $upload_destination" >&2 - exit 1 -} -DONE + # Tell gencode-remote to where we uploaded the file so it can move # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" From 8b915b15b5a969b7de308e2b1b124d28548349dc Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 14 Apr 2022 00:46:13 +0200 Subject: [PATCH 46/77] __file: make the create-empty-file case work again Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index aa7149c1..f9acdb17 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -72,6 +72,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 + echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -88,6 +89,14 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" + if [ "$create_file" ]; then + # When creating an empty file we create it locally and then + # upload it so that permissions can be set before moving the file + # into place. + source="$__object/files/empty" + touch "$source" + fi + # upload file to temp location upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" # Yes, we are aware that this is a race condition. @@ -104,18 +113,15 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then # it to its final destination. echo "$upload_destination" > "$__object/files/upload-destination" - if [ "$upload_file" ]; then - echo upload >> "$__messages_out" - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi - cat << DONE + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE $__remote_copy "$source" "${my_target_host}:${upload_destination}" DONE - fi fi fi From abbc7dfc376ffb6554bfaad2d5a6ec93955f235f Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 16 Apr 2022 19:05:31 +0200 Subject: [PATCH 47/77] since we already remove the destination, we have no need to use -T on move, fixes #333 Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 136520a7..1a9ff69c 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -91,7 +91,7 @@ case "$state_should" in if [ -f "$__object/files/upload-destination" ]; then # move uploaded file into place printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv -T "%s" "%s"\n' "$destination" "$final_destination" + printf 'mv "%s" "%s"\n' "$destination" "$final_destination" fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local From 6c8c692a22c886bf82b18f51a133f88a8ab547b7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 2 May 2022 23:25:59 +0200 Subject: [PATCH 48/77] __file: kiss and fix regression on Mac OSX Signed-off-by: Steven Armstrong --- cdist/conf/type/__file/gencode-local | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index f9acdb17..5a303308 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -98,7 +98,7 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then fi # upload file to temp location - upload_destination="$(mktemp -u "/__cdist${destination}.cdist.XXXXXXXXXX" | sed 's|^/__cdist||')" + upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" # Yes, we are aware that this is a race condition. # However: # a) cdist usually writes to directories that are not user writable From 3d58c9b24fede2d1dafb15fc36424c52c466bb0d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Fri, 20 May 2022 13:48:07 +0200 Subject: [PATCH 49/77] add optional file parameter to allow for use in a loop without object_id clashes --- cdist/conf/type/__dot_file/man.rst | 15 +++++++++++++++ cdist/conf/type/__dot_file/manifest | 10 ++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index ba7621a1..c8f36712 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,6 +37,12 @@ state source forwarded to :strong:`__file` type +file + forwarded to :strong:`__file` type + This can be used if multiple users need to have a dotfile updated, + which will result in duplicate object id errors. When using the + file parameter the object id can be some unique value. + MESSAGES -------- @@ -61,6 +67,15 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" + # install .vimrc for root and some users + for user in root userx usery userz; do + __dot_file "${user}_dot_vimrc" \ + --user $user \ + --file .vimrc \ + --state exists \ + --source "$__files/$user/.vimrc" + done + SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 02dadf05..a38ed943 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,13 +20,19 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" +if [ -f "${__object}/parameter/file" ]; then + file="$(cat "${__object}/parameter/file")" +else + file="${__object_id}" +fi + # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${__object_id} +subpath=${file} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -64,4 +70,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" From 9839c2d8ec63719c30493870851f1e1ab8079fb1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:55:12 +0200 Subject: [PATCH 50/77] ++changelog Signed-off-by: Nico Schottelius --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3363be27..29507443 100644 --- a/docs/changelog +++ b/docs/changelog @@ -15,6 +15,7 @@ next: * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) + * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 77d9a757ec0e6986388aa368b77f065ae6be68c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 20 May 2022 14:58:45 +0200 Subject: [PATCH 51/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 29507443..8c9e1028 100644 --- a/docs/changelog +++ b/docs/changelog @@ -16,6 +16,7 @@ next: * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) + * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 0ae37b3445c6662a968c49972e0102c75d6f14c2 Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 1 Jul 2022 14:27:33 +0100 Subject: [PATCH 52/77] Handle signed-by option in __apt_source Allow users to specify a GPG key fingerprint or keyring file to be included as the 'signed-by' option. --- .../type/__apt_source/files/source.list.template | 3 ++- cdist/conf/type/__apt_source/man.rst | 8 ++++++++ cdist/conf/type/__apt_source/manifest | 14 ++++++++++---- cdist/conf/type/__apt_source/parameter/optional | 3 ++- 4 files changed, 22 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index d4420e96..a28bb45f 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,13 +2,14 @@ set -u entry="$uri $distribution $component" + cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${forcedarch} $entry +deb ${options} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d1acb388..d317a135 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,6 +23,9 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) +signed-by + provide a GPG key fingerprint or keyring path for signature checks + state 'present' or 'absent', defaults to 'present' @@ -56,6 +59,11 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present + __apt_source goaccess \ + --uri http://deb.goaccess.io/ \ + --component main \ + --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D + AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index 35f15909..cdb526d3 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -31,9 +31,15 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - forcedarch="[arch=$(cat "$__object/parameter/arch")]" -else - forcedarch="" + options="arch=$(cat "$__object/parameter/arch")" +fi + +if [ -f "$__object/parameter/signed-by" ]; then + options="$options signed-by=$(cat "$__object/parameter/signed-by")" +fi + +if [ "$options" ]; then + options="[$options]" fi # export variables for use in template @@ -41,7 +47,7 @@ export name export uri export distribution export component -export forcedarch +export options # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 87537335..0b5470a1 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,4 +1,5 @@ state distribution component -arch \ No newline at end of file +arch +signed-by From 339ca9347b8dc3d9167393bf754a92c4dedbd704 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 2 Jul 2022 19:21:27 +0200 Subject: [PATCH 53/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 8c9e1028..3d9cb704 100644 --- a/docs/changelog +++ b/docs/changelog @@ -17,6 +17,7 @@ next: * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) + * Type __apt_source: Add signed-by parameter (Daniel Fancsali) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From 483f0c161440a018efe48bc96910dcb5834c32a1 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Wed, 13 Jul 2022 14:50:17 +0200 Subject: [PATCH 54/77] add Check Point Gaia --- cdist/conf/explorer/lsb_codename | 3 +++ cdist/conf/explorer/lsb_description | 3 +++ cdist/conf/explorer/lsb_id | 3 +++ cdist/conf/explorer/lsb_release | 3 +++ cdist/conf/explorer/os | 7 +++++++ cdist/conf/explorer/os_release | 4 ++++ cdist/conf/explorer/os_version | 3 +++ 7 files changed, 26 insertions(+) diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index 26bb8e3d..c9fb5cdf 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + awk '{printf("%s\n", $(NF-1))}' /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index b1009627..7279a9c2 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 82ff9977..1f91cc40 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + echo "CheckPoint" + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 5ebfff1a..d9100569 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,6 +21,9 @@ set +e case "$("$__explorer/os")" in + checkpoint) + cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 46d87f3e..b9232ee4 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,6 +116,13 @@ if [ -f /etc/slackware-version ]; then exit 0 fi +# Appliances + +if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then + echo checkpoint + exit 0 +fi + uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index 6489446b..ec85046f 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,5 +34,9 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release +elif test -f /etc/cp-release +then + # Checkpoint firewall or management (actually linux based) + cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index bbc9e4f0..430200ae 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -41,6 +41,9 @@ in # empty, but well... cat /etc/arch-release ;; + checkpoint) + awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release + ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version From 624316564531cc7575d2ee0504e3a5f684c74e3d Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 16:27:12 +0200 Subject: [PATCH 55/77] add create and ifexists to line type --- cdist/conf/type/__line/gencode-remote | 12 ++++++++++-- cdist/conf/type/__line/man.rst | 6 ++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index a89886da..95c2360d 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,8 +34,16 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 + if [ -f "$__object/parameter/create" ]; then + echo "touch $file" + else + # only bark if the file should exists + if [ -f "$__object/parameter/ifexists" ]; then + exit 0 + fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 + fi fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index 70490f68..dd41fc40 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,6 +21,9 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. +create + It the file does not exist then create an empty file + before Insert the given line before this pattern. @@ -28,6 +31,9 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. +ifexists + Only apply the line if the file exists. + line Specifies the line which should be absent or present. From 7d8fc8a5c38c47a36ea87cd98d798d22f91c3185 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:18:41 +0200 Subject: [PATCH 56/77] improve checkpoint sed, add __line changes --- cdist/conf/explorer/lsb_release | 2 +- cdist/conf/type/__line/parameter/boolean | 2 ++ cdist/conf/type/__line/parameter/default/create | 1 + cdist/conf/type/__line/parameter/default/ifexists | 1 + 4 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__line/parameter/boolean create mode 100644 cdist/conf/type/__line/parameter/default/create create mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index d9100569..0bb9f7fe 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -22,7 +22,7 @@ set +e case "$("$__explorer/os")" in checkpoint) - cat /etc/cp-release|sed -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' + sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' ;; openwrt) # shellcheck disable=SC1091 diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean new file mode 100644 index 00000000..182a5da6 --- /dev/null +++ b/cdist/conf/type/__line/parameter/boolean @@ -0,0 +1,2 @@ +create +ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/create @@ -0,0 +1 @@ +false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists new file mode 100644 index 00000000..c508d536 --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/ifexists @@ -0,0 +1 @@ +false From 17466452f0762fc3a259a66742a59943c5af2510 Mon Sep 17 00:00:00 2001 From: Stephan Leemburg Date: Thu, 28 Jul 2022 17:53:41 +0200 Subject: [PATCH 57/77] revert __line for clean PR history --- cdist/conf/type/__line/gencode-remote | 12 ++---------- cdist/conf/type/__line/man.rst | 6 ------ cdist/conf/type/__line/parameter/boolean | 2 -- cdist/conf/type/__line/parameter/default/create | 1 - cdist/conf/type/__line/parameter/default/ifexists | 1 - 5 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 cdist/conf/type/__line/parameter/boolean delete mode 100644 cdist/conf/type/__line/parameter/default/create delete mode 100644 cdist/conf/type/__line/parameter/default/ifexists diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index 95c2360d..a89886da 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -34,16 +34,8 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" if [ -z "$state_is" ]; then - if [ -f "$__object/parameter/create" ]; then - echo "touch $file" - else - # only bark if the file should exists - if [ -f "$__object/parameter/ifexists" ]; then - exit 0 - fi - printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 - exit 1 - fi + printf 'The file "%s" is missing. Please create it before using %s on it.\n' "$file" "${__type##*/}" >&2 + exit 1 fi if [ "$state_should" = "$state_is" ] || \ diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index dd41fc40..70490f68 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -21,9 +21,6 @@ OPTIONAL PARAMETERS after Insert the given line after this pattern. -create - It the file does not exist then create an empty file - before Insert the given line before this pattern. @@ -31,9 +28,6 @@ file If supplied, use this as the destination file. Otherwise the object_id is used. -ifexists - Only apply the line if the file exists. - line Specifies the line which should be absent or present. diff --git a/cdist/conf/type/__line/parameter/boolean b/cdist/conf/type/__line/parameter/boolean deleted file mode 100644 index 182a5da6..00000000 --- a/cdist/conf/type/__line/parameter/boolean +++ /dev/null @@ -1,2 +0,0 @@ -create -ifexists diff --git a/cdist/conf/type/__line/parameter/default/create b/cdist/conf/type/__line/parameter/default/create deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/create +++ /dev/null @@ -1 +0,0 @@ -false diff --git a/cdist/conf/type/__line/parameter/default/ifexists b/cdist/conf/type/__line/parameter/default/ifexists deleted file mode 100644 index c508d536..00000000 --- a/cdist/conf/type/__line/parameter/default/ifexists +++ /dev/null @@ -1 +0,0 @@ -false From d4bf41ce3b7d186e7e85c030c5960ddd38855a29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Jul 2022 10:57:01 +0200 Subject: [PATCH 58/77] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3d9cb704..db2faef0 100644 --- a/docs/changelog +++ b/docs/changelog @@ -18,6 +18,7 @@ next: * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) * Type __apt_source: Add signed-by parameter (Daniel Fancsali) + * Explorer: add support for checkpoint (Stephan Leemburg) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) From be6e7fcc08d5c4e98f7ce30b157ee087010cd126 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 31 Jul 2022 21:59:35 +0200 Subject: [PATCH 59/77] Prepare release of cdist 7.0.0 --- bin/cdist-build-helper | 2 +- docs/changelog | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index 0380b3f8..cadddae7 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. diff --git a/docs/changelog b/docs/changelog index db2faef0..00defc2a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) * New type: __haproxy_dualstack (Evilham and ungleich) From 90488d2e9e21b4e6bf3b7f0d8c44ebfe29c517b7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Aug 2022 00:03:51 +0200 Subject: [PATCH 60/77] [doc] add release process documentation --- docs/dev/release-process.org | 90 ++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 docs/dev/release-process.org diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org new file mode 100644 index 00000000..42b4f5c5 --- /dev/null +++ b/docs/dev/release-process.org @@ -0,0 +1,90 @@ +* Install requirements (Alpine) + - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ + py3-build twine +* Ensure your gpg setup works with the email used in the git commit! + - For me this is nico@nico-notebook.schottelius.org + - Signature / id is on nb2 +* Create ~/.pypirc +[distutils] + index-servers = + pypi + cdist + +[pypi] + username = __token__ + password = ... + +[cdist] + repository = https://upload.pypi.org/legacy/ + username = __token__ + password = ... + +* Add date in docs/changelog +* Run ./bin/cdist-build-helper +* TODO Move to "build" + - python3 -m build +* DONE git tag: when? +CLOSED: [2022-07-31 Sun 23:58] +** Asked during release process: ok +* DONE Pypi error with distutils: do not use distutils anymore +CLOSED: [2022-07-31 Sun 23:58] +python3 setup.py sdist upload +... +Creating tar archive +removing 'cdist-7.0.0' (and everything under it) +running upload +Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ +Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. +(venv2) [22:50] nb2:cdist% + +* DONE Pypi error with twine: fixed in twine 4.0.1 +CLOSED: [2022-07-31 Sun 23:58] + +Seeing: + +(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* +Uploading distributions to https://upload.pypi.org/legacy/ +Traceback (most recent call last): + File "/usr/bin/twine", line 8, in + sys.exit(main()) + File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main + result = cli.dispatch(sys.argv[1:]) + File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch + return main(args.args) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main + return upload(upload_settings, parsed_args.dists) + File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload + resp = repository.upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload + resp = self._upload(package) + File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload + data = package.metadata_dictionary() + File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary + "dynamic": meta.dynamic, +AttributeError: 'Wheel' object has no attribute 'dynamic' + + +Fix: + + +(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* +⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. +Uploading distributions to https://upload.pypi.org/legacy/ +Uploading cdist-7.0.0-py3-none-any.whl +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s +Uploading cdist-7.0.0.tar.gz +100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s + +View at: +https://pypi.org/project/cdist/7.0.0/ +* TODO cdist web + - on staticweb-2022 + - Should be moved to sftp/k8s + + + Manual steps: + + ~/bin/permissions.public html/ + rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ + ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest" From c85184dcb48118ea5edbda2799badb5779253f7c Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Sun, 18 Sep 2022 08:49:37 +0200 Subject: [PATCH 61/77] Make sure flag is followed by end of line or space. --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 00646c75..0f212e73 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -515,7 +515,7 @@ check_vm_arch_specific() { && return 0 fi if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\) /proc/cpuinfo then return 0 fi From 62db96bb376354c829cdde290c915cffc24612b0 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Thu, 29 Sep 2022 16:19:07 +0200 Subject: [PATCH 62/77] Initialise options variable so expansion when running files/source.list.template there will not be an error when the variable is not set. --- cdist/conf/type/__apt_source/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index cdb526d3..fd1ec47f 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -21,6 +21,7 @@ name="$__object_id" state="$(cat "$__object/parameter/state")" uri="$(cat "$__object/parameter/uri")" +options="" if [ -f "$__object/parameter/distribution" ]; then distribution="$(cat "$__object/parameter/distribution")" From ffeaafe9b68a944418ec8496c7db9f347dd4b1ee Mon Sep 17 00:00:00 2001 From: marcoduif Date: Fri, 7 Oct 2022 07:22:31 +0000 Subject: [PATCH 63/77] Make grep more specific package name should be an exact match, not a substring --- cdist/conf/type/__apt_mark/explorer/state | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state index b7fe08fa..b464179a 100755 --- a/cdist/conf/type/__apt_mark/explorer/state +++ b/cdist/conf/type/__apt_mark/explorer/state @@ -24,4 +24,4 @@ else name="$__object_id" fi -apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold +apt-mark showhold | grep -q "^${name}$" && echo hold || echo unhold From f36069754c0101d1eccbef59fa4f060a19784d26 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Dec 2022 18:03:15 +0100 Subject: [PATCH 64/77] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 00defc2a..af41368d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +7.0.1: + * Type __apt_mark: Narrow down grep for hold packages (marcoduif) + 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) * New type: __sed (Ander Punnar) From bdfd92dc3707004503af4a03aea0819ad587a947 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 21 Dec 2022 09:41:33 +0100 Subject: [PATCH 65/77] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index af41368d..e8090131 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog 7.0.1: * Type __apt_mark: Narrow down grep for hold packages (marcoduif) + * Type __apt_source: Set required options variable (Mark Verboom) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From b974969f28f4d007f75904757e80e4f663e3d134 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Dec 2022 20:59:16 +0100 Subject: [PATCH 66/77] Remove double definition of scan parser Fixes #353 --- cdist/argparse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index f17315e7..8f7bbb85 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -472,9 +472,6 @@ def get_parsers(): parser['info'].set_defaults(func=cdist.info.Info.commandline) # Scan = config + further - parser['scan'] = parser['sub'].add_parser('scan', add_help=False, - parents=[parser['config']]) - parser['scan'] = parser['sub'].add_parser( 'scan', parents=[parser['loglevel'], parser['beta'], From ed3da3c829d47c7c578733482920f111e8bc8869 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Dec 2022 21:02:41 +0100 Subject: [PATCH 67/77] ++changes --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e8090131..6ade5d47 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog 7.0.1: * Type __apt_mark: Narrow down grep for hold packages (marcoduif) * Type __apt_source: Set required options variable (Mark Verboom) + * Core: Remove double definition of scan parser (Nico Schottelius) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From 08a6b467fa7afcd2f2a0a2497441cee773166d6e Mon Sep 17 00:00:00 2001 From: Michelle Date: Wed, 25 Jan 2023 16:06:35 -0500 Subject: [PATCH 68/77] Added support for Devuan Daedalus Added one line that allows cdist to support Devuan Daedelus version --- cdist/conf/explorer/os_version | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 430200ae..fc59fd14 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -82,6 +82,7 @@ in # ceres versions don't have a number, so we decode by codename: case ${devuan_version} in + (daedalus/ceres) echo 4.99 ;; (chimaera/ceres) echo 3.99 ;; (beowulf/ceres) echo 2.99 ;; (ascii/ceres) echo 1.99 ;; From 1450861e26c8c5dd249517fa14fa8ea0f4961d3d Mon Sep 17 00:00:00 2001 From: Michelle Date: Thu, 2 Feb 2023 18:21:43 -0500 Subject: [PATCH 69/77] Updated the python version of cerbot freebsd The package referenced for cerbot to be install in the FreeBSD platform used python 3.7 package, updated to python 3.9 --- cdist/conf/type/__letsencrypt_cert/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 638a99e0..39067f3b 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -85,7 +85,7 @@ if [ -z "${certbot_fullpath}" ]; then esac ;; freebsd) - __package py37-certbot + __package py39-certbot certbot_fullpath="/usr/local/bin/certbot" ;; ubuntu) From 7dd2d1025ae3b005e5d44d11e1ca62536119f36c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 3 Feb 2023 22:54:13 +0100 Subject: [PATCH 70/77] ++changelog --- docs/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 6ade5d47..4e9a8a6d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,9 +2,12 @@ Changelog --------- 7.0.1: + * Core: Remove double definition of scan parser (Nico Schottelius) * Type __apt_mark: Narrow down grep for hold packages (marcoduif) * Type __apt_source: Set required options variable (Mark Verboom) - * Core: Remove double definition of scan parser (Nico Schottelius) + * Type __letsencrypt_cert: Update python version (Michelle) + * Explorer os_version: Add support for Daedalus (Michelle) + * Explorer machine_type: Correct incorrect VMM matching (Mark Verboom) 7.0.0: 2022-07-31 * Explorer machine_type: Rewrite (Dennis Camera) From e57cf1e70a3818180b3f54e1e0364ff3b6bbd9cc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 25 Mar 2023 12:53:24 +0100 Subject: [PATCH 71/77] Apply machine_type explorer fix from pedro https://code.ungleich.ch/pedro/cdist/commit/e13939752913eea29b84d6e4659e1ca9e0e9ffeb --- cdist/conf/explorer/machine_type | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 0f212e73..c31f5ca6 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -515,7 +515,7 @@ check_vm_arch_specific() { && return 0 fi if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\) /proc/cpuinfo + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\)' /proc/cpuinfo then return 0 fi From 2b102f303a9c2a49e1171cddb1b7fb86309ab5e9 Mon Sep 17 00:00:00 2001 From: marcoduif Date: Wed, 20 Dec 2023 08:01:02 +0000 Subject: [PATCH 72/77] changed package state from installed to present --- cdist/conf/type/__package_luarocks/manifest | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__package_luarocks/manifest b/cdist/conf/type/__package_luarocks/manifest index 7d8262ca..9e4499b2 100755 --- a/cdist/conf/type/__package_luarocks/manifest +++ b/cdist/conf/type/__package_luarocks/manifest @@ -19,5 +19,5 @@ # along with cdist. If not, see . # -__package luarocks --state installed -__package make --state installed +__package luarocks --state present +__package make --state present From 61fc5e5de8adeb6d2e49c37b2cbb64c750edcb0b Mon Sep 17 00:00:00 2001 From: Daniel Fancsali Date: Fri, 8 Jul 2022 16:50:54 +0100 Subject: [PATCH 73/77] Fix typo in __apt_pin docs --- cdist/conf/type/__apt_pin/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index 4229c0cd..e6ec8b51 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -23,7 +23,7 @@ package Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority - The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) + The priority value to assign to matching packages. Defaults to 500. (To match the default target distro's priority) state Will be passed to underlying `__file` type; see there for valid values and defaults. From b7394ff4c25846be5ad8bd4fa793ceda9efae5f3 Mon Sep 17 00:00:00 2001 From: Mark Verboom Date: Thu, 2 Nov 2023 11:01:45 +0100 Subject: [PATCH 74/77] Locally sort remote group information, to prevent differences in sort output. --- cdist/conf/type/__user_groups/explorer/group | 2 +- cdist/conf/type/__user_groups/gencode-remote | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__user_groups/explorer/group b/cdist/conf/type/__user_groups/explorer/group index 5bad9a0b..8a02f219 100755 --- a/cdist/conf/type/__user_groups/explorer/group +++ b/cdist/conf/type/__user_groups/explorer/group @@ -20,4 +20,4 @@ user="$(cat "$__object/parameter/user" 2>/dev/null || echo "$__object_id")" -(id -G -n "$user" | tr ' ' '\n' | sort) 2>/dev/null || true +(id -G -n "$user" | tr ' ' '\n') 2>/dev/null || true diff --git a/cdist/conf/type/__user_groups/gencode-remote b/cdist/conf/type/__user_groups/gencode-remote index 8120761a..0585e90f 100755 --- a/cdist/conf/type/__user_groups/gencode-remote +++ b/cdist/conf/type/__user_groups/gencode-remote @@ -26,13 +26,15 @@ os=$(cat "$__global/explorer/os") mkdir "$__object/files" # file has to be sorted for comparison with `comm` sort "$__object/parameter/group" > "$__object/files/group.sorted" +# Use local sort for remote groups +sort "$__object/explorer/group" > "$__object/files/group-remote.sorted" case "$state_should" in present) - changed_groups="$(comm -13 "$__object/explorer/group" "$__object/files/group.sorted")" + changed_groups="$(comm -13 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")" ;; absent) - changed_groups="$(comm -12 "$__object/explorer/group" "$__object/files/group.sorted")" + changed_groups="$(comm -12 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")" ;; esac From 3e82b0085b693e6b23f2b613e2f1dd975d67e6dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C4=BDubom=C3=ADr=20Ku=C4=8Dera?= Date: Wed, 1 May 2024 12:11:07 +0000 Subject: [PATCH 75/77] Make development version strings PEP 440 compliant (#366) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With current pip (23.1.2) and setuptools (67.7.2) versions, installation from Git was failing with "Invalid version: '7.0.0-17-ge57cf1e7'" message. The issue can be reproduced by running the following command with the latest pip and setuptools installed: $ pip install git+https://code.ungleich.ch/ungleich-public/cdist.git@e57cf1e70a3818180b3f54e1e0364ff3b6bbd9cc#egg=cdist Reviewed-on: https://code.ungleich.ch/ungleich-public/cdist/pulls/366 Co-authored-by: Ľubomír Kučera Co-committed-by: Ľubomír Kučera --- bin/cdist-build-helper | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index cadddae7..6f514ef5 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -534,7 +534,8 @@ eof ;; version) - printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py + target_version="$(git describe | sed 's/-/.dev/; s/-/+/g')" + printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py ;; target-version) From 27471a4a82ac29c1a1f3062178a509660d55e732 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 2 Aug 2024 12:05:37 +0200 Subject: [PATCH 76/77] [__timezone] add support for openwrt --- cdist/conf/type/__timezone/gencode-remote | 9 +++++++++ cdist/conf/type/__timezone/manifest | 5 ++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote index b685c990..d8612986 100755 --- a/cdist/conf/type/__timezone/gencode-remote +++ b/cdist/conf/type/__timezone/gencode-remote @@ -34,3 +34,12 @@ case "$os" in echo "echo \"$timezone_should\" > /etc/timezone" ;; esac + +case "$os" in + openwrt) + cat <