diff --git a/Makefile b/Makefile index 89286310..3712511c 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man -SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html -SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean +SPHINXM=make -C $(DOCS_SRC_DIR) man +SPHINXH=make -C $(DOCS_SRC_DIR) html +SPHINXC=make -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/README.md b/README.md index a468dd86..de6901c7 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ [libera](https://libera.chat) +IRC: ``#cdist`` @ freenode Matrix: ``#cdist:ungleich.ch`` -Matrix and IRC are bridged. +Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist diff --git a/bin/cdist b/bin/cdist index adb06a8d..ddaffa7f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -72,11 +72,9 @@ def commandline(): if __name__ == "__main__": - if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION: - print( - 'Python >= {} is required on the source host.'.format( - ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))), - file=sys.stderr) + if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: + print('Python >= {} is required on the source host.'.format( + cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) sys.exit(1) exit_code = 0 diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index cadddae7..0380b3f8 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. diff --git a/cdist/__init__.py b/cdist/__init__.py index 31d49889..44366cd0 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" -MIN_SUPPORTED_PYTHON_VERSION = (3, 5) +MIN_SUPPORTED_PYTHON_VERSION = '3.5' class Error(Exception): diff --git a/cdist/argparse.py b/cdist/argparse.py index f17315e7..cadac39a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,31 +485,19 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger', 'config']) - parser['scan'].add_argument( - '--list', - action='store_true', - help='List the known hosts and exit') + choices=['scan', 'trigger']) parser['scan'].add_argument( '--config', action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interface', - action='append', default=[], required=True, + '-I', '--interfaces', + action='append', default=[], help='On which interfaces to scan/trigger') parser['scan'].add_argument( - '--name-mapper', - action='store', default=None, - help='Map addresses to names, required for config mode') - parser['scan'].add_argument( - '-d', '--config-delay', - action='store', default=3600, type=int, - help='How long (seconds) to wait before reconfiguring after last try') - parser['scan'].add_argument( - '-t', '--trigger-delay', - action='store', default=5, type=int, - help='How long (seconds) to wait between ICMPv6 echo requests') + '-d', '--delay', + action='store', default=3600, + help='How long to wait before reconfiguring after last try') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index c9fb5cdf..26bb8e3d 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - awk '{printf("%s\n", $(NF-1))}' /etc/cp-release - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index 7279a9c2..b1009627 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - cat /etc/cp-release - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 1f91cc40..82ff9977 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - echo "CheckPoint" - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 0bb9f7fe..5ebfff1a 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 00646c75..1c84f4d7 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,6 +1,8 @@ -#!/bin/sh -e +#!/bin/sh # -# 2021 Dennis Camera (cdist at dtnr.ch) +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# 2020 Evilham (contact at evilham.com) # # This file is part of cdist. # @@ -17,1019 +19,91 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer tries to determine what type of machine the target to be -# configured is (container, virtual machine, bare-metal). -# -# It will print one line for each layer it can detect. -# The format of all lines is: TYPE[ VERB VENDOR] -# -# VERB does not have a special meaning, it is just for better readability. -# -# e.g. -# container -# container on lxc -# virtual by kvm-spapr -# -# The third word of each line (except the first) can be composed of different -# parts concatenated with a `-' (minus) character, with each component being -# a specification of the previous, e.g.: -# - lxc-libvirt (LXC container, managed by libvirt) -# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) -# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) -# -# If this explorer cannot collect enough information about virtualization it -# will fall back to 'physical'. -# -# Add /sbin and /usr/sbin to the path so we can find system -# binaries like dmidecode. -PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' -PATH="/sbin:/usr/sbin:${PATH}" -export PATH +os=$("$__explorer/os") -arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') -uname_s=$(uname -s) - - -is_command() { command -v "$1" >/dev/null 2>&1; } - -files_same() { - # shellcheck disable=SC2012 - LC_ALL=C df -P "$1" "$2" 2>/dev/null | { - read -r _ # skip header line - read -r fs1 _ _ _ _ mp1 - read -r fs2 _ _ _ _ mp2 - test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 - } && - ls -1Ldi "$1" "$2" 2>/dev/null | { - read -r ino1 _ - read -r ino2 _ - test "${ino1}" = "${ino2}" || return 1 - } +vendor_string_to_machine_type() { + for vendor in vmware bochs kvm qemu virtualbox bhyve; do + if echo "${1}" | grep -q -i "${vendor}"; then + if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then + vendor="kvm" + fi + echo "virtual_by_${vendor}" + exit + fi + done } -is_oneof() ( - x=$1; shift - for y - do - test "${x}" = "${y}" || continue - return 0 - done - return 1 -) - -tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } - -# shellcheck disable=SC2086 -glob_exists() { set -- $1; test -e "$1"; } - -get_dmi_field() { - if is_oneof "${uname_s}" NetBSD - then - case $1 - in - (system-manufacturer) _mib=machdep.dmi.system-vendor ;; - (system-product-name) _mib=machdep.dmi.system-product ;; - (system-version|system-uuid) _mib=machdep.dmi.$1 ;; - (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; - (biod-release-date) _mib=machdep.dmi.bios-date ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - if is_command dmidecode - then - dmidecode -s "$1" - elif test -d "${dmi_sysfs-}" - then - case $1 - in - (system-manufacturer) _filename=sys_vendor ;; - (system-product-name) _filename=product_name ;; - (*) _filename=$(echo "$1" | tr - _) ;; - esac - if test -r "${dmi_sysfs-}/${_filename}" - then - cat "${dmi_sysfs}/${_filename}" - fi - unset _filename - elif test "${uname_s}" = OpenBSD - then - # NOTE: something similar to system-manufacutrer and system-product-name - # is available on OpenBSD in sysctl - case $1 - in - (system-manufacturer) _mib=hw.vendor ;; - (system-product-name) _mib=hw.product ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - return 1 -} - -has_cpuinfo() { test -e /proc/cpuinfo; } - -get_sysctl() { - is_command sysctl && sysctl -n "$1" 2>/dev/null -} - -detected_layer() { - test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" -} - - -# Check for chroot - -has_chroot_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' -} - -check_chroot_systemd() { - systemd-detect-virt -r -} - -has_chroot_debian_ischroot() { - is_command ischroot -} - -check_chroot_debian_ischroot() { - ischroot --default-false -} - -has_chroot_procfs() { - test -d /proc/ -} - -check_chroot_procfs() ( - is_chroot=false # default - if test -e /proc/1/root && ! files_same /proc/1/root / - then - is_chroot=true - fi - if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo - then - has_mountinfo=true - cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true - fi - - if ${is_chroot} - then - # try to determine where the chroot has been mounted - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # escape chroot and find mount point by inode - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - elif ${has_mountinfo} - then - while read -r mntid _ _ _ cmntpnt _ - do - read -r _ _ _ _ hmntpnt _ <<-EOF - $(grep -e "^$((mntid)) " /proc/1/mountinfo) - EOF - printf '%s\n' "${hmntpnt%${cmntpnt}}" - done /dev/null) && - case ${_ctengine} - in - (''|'none') - return 1 ;; - ('container-other') - return 0 ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - echo "${_ctengine}" ;; - esac -) - -has_ct_pid_1() { - test -r /run/systemd/container -o -r /proc/1/environ -} - -translate_container_name() { - case $1 - in - ('lxc') - echo lxc ;; - ('lxc-libvirt') - echo lxc-libvirt ;; - ('podman') - echo podman ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - return 1 ;; - esac - return 0 -} - -check_ct_pid_1() { - if test -r /run/systemd/container - then - translate_container_name "$(head -n1 /run/systemd/container)" \ - && return 0 - fi - - if test -r /proc/1/environ - then - translate_container_name "$( - LC_ALL=C tr '\000' '\n' /dev/null - then - # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 - echo wsl - elif test -d /var/.cagefs - then - # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs - # CageFS is not "really" a container, but it isn't a chroot either. - echo cagefs - elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status - then - # Linux-VServer - if grep -q -x -F 'VxID: 0' /proc/self/status - then - # host - return 1 - else - # guest - echo linux_vserver - fi - else - return 1 - fi -} - -check_ct_os_specific() ( - if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 - then - # FreeBSD jail - echo jail - return 0 - fi - - if is_command zonename && test "$(zonename)" != global - then - # Solaris zone - echo zone - return 0 - fi - - return 1 -) - - -# Check for hypervisor - -guess_hypervisor_from_cpu_model() { - case $1 - in - (*\ KVM\ *) - echo kvm ;; - (*\ QEMU\ *|QEMU\ *) - echo qemu ;; - (*) - return 1 ;; - esac -} - -has_vm_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' -} - -check_vm_systemd() ( - _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && - case ${_hypervisor} - in - (''|'none') - return 1 ;; - ('amazon') - echo aws ;; - ('bochs') - echo kvm ;; - ('microsoft') - # assumption - echo hyperv ;; - ('oracle') - echo virtualbox ;; - ('vm-other') - return 0 ;; - (*) - echo "${_hypervisor}" ;; - esac -) - -has_vm_cpuinfo() { has_cpuinfo; } - -check_vm_cpuinfo() { - if grep -q -F 'User Mode Linux' /proc/cpuinfo \ - || grep -q -F 'UML' /proc/cpuinfo - then - # User Mode Linux - echo uml - elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo - then - # IBM PowerVM Lx86 (Linux/x86 emulator) - echo powervm_lx86 - elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo - then - # IBM SystemZ (S/390) - if test -f /proc/sysinfo - then - if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo - then - echo kvm-s390 - return 0 - elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo - then - echo zvm - return 0 - elif grep -q -e '^LPAR ' /proc/sysinfo - then - echo zvm-lpar - return 0 - fi - fi - return 1 - else - if grep -q -e '^model name.*:' /proc/cpuinfo - then - sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ - | while read -r _cpu_model - do - guess_hypervisor_from_cpu_model "${_cpu_model}" - done \ - | sort \ - | uniq -c \ - | awk ' - { if ($1 > most_c) { most_c = $1; most_s = $2 } } - END { - if (most_s) print most_s - exit !most_s - }' \ - && return 0 - fi - return 1 - fi -} - -check_vm_arch_specific() { - case ${arch} - in - (ppc64|ppc64le) - # Check PPC64 LPAR, KVM - - # example /proc/cpuinfo line indicating 'not baremetal' - # platform : pSeries - # - # example /proc/ppc64/lparcfg systemtype line - # system_type=IBM pSeries (emulated by qemu) - - if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo - then - if test -e /proc/ppc64/lparcfg - then - # Assume LPAR, now detect shared or dedicated - if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg - then - echo powervm-shared - return 0 - else - echo powervm-dedicated - return 0 - fi - fi - fi - ;; - (sparc*) - # Check for SPARC LDoms - - if test -e /dev/mdesc - then - if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp - then - # control LDom - return 1 - else - # guest LDom - echo ldom-sparc - fi - - # MDPROP=/usr/lib/ldoms/mdprop.py - # if test -x "${MDPROP}" - # then - # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" - # then - # echo ldoms-root - # echo ldoms-io - # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" - # then - # echo ldoms-io - # fi - # fi - return 0 - fi - ;; - (i?86|x86*|amd64|i86pc) - # VMM CPUID flag denotes that this system is running under a VMM - if is_oneof "${uname_s}" Darwin - then - get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ - && return 0 - fi - if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo - then - return 0 - fi - ;; - (ia64) - if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend - then - # PV-on-HVM drivers installed in a Xen guest - echo xen-hvm - return 0 - fi - ;; - esac - return 1 -} - -has_vm_dmi() { - # Check for various products in SMBIOS/DMI. - # Note that DMI doesn't exist on all architectures (only x86 and some ARM). - # On other architectures the $dmi variable will be empty. - - if test -d /sys/class/dmi/id/ - then - dmi_sysfs=/sys/class/dmi/id - elif test -d /sys/devices/virtual/dmi/id/ - then - dmi_sysfs=/sys/devices/virtual/dmi/id - fi - - # shellcheck disable=SC2015 - { - is_command dmidecode \ - && ( - # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line - dmi_out=$(dmidecode 2>&1) \ - && ! printf '%s\n' "${dmi_out}" \ - | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' - ) \ - || test -d "${dmi_sysfs}" - } -} - -check_vm_dmi() { - case $(get_dmi_field system-product-name) - in - (*.metal) - if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' - then - # AWS EC2 bare metal -> no virtualisation - return 1 - fi - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('Google Compute Engine') - echo gce - return 0 - ;; - ('RHEV Hypervisor') - # Red Hat Enterprise Virtualization - echo rhev - return 0 - ;; - ('KVM'|'Bochs'|'KVM Virtual Machine') - echo kvm - return 0 - ;; - ('Parallels Virtual Platform') - echo parallels - return 0 - ;; - ('VirtualBox') - echo virtualbox - return 0 - ;; - ('VMware Virtual Platform') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field system-manufacturer) - in - ('Alibaba'*) - case $(get_dmi_field system-product-name) - in - ('Alibaba Cloud ECS') - echo alibaba-ecs - ;; - (*) - echo alibaba - ;; - esac - return 0 - ;; - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('innotek GmbH'|'Oracle Corporation') - echo virtualbox - return 0 - ;; - ('Joyent') - if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' - then - # SmartOS KVM - echo kvm-smartdc_hvm - return 0 - fi - ;; - ('Microsoft Corporation'*) - if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' - then - if test -e /proc/irq/7/hyperv \ - || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null - then - echo hyperv - return 0 - fi - - case $(get_dmi_field system-version) - in - (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) - echo virtualpc - return 0 - ;; - (*) - echo hyperv - return 0 - ;; - esac - fi - ;; - ('Nutanix') - # Nutanix AHV. Similar to KVM. - if test "$(get_dmi_field system-product-name)" = 'AHV' - then - echo nutanix_ahv - return 0 - fi - ;; - ('oVirt') - echo ovirt - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('QEMU') - echo qemu - return 0 - ;; - ('VMware, Inc.') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field bios-vendor) - in - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('innotek GmbH') - echo virtualbox - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('Xen') - if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' - then - # AWS on Xen - echo aws-xen - return 0 - fi - ;; - esac - - return 1 -} - -check_vm_hyp_specific() { - if is_command vmware-checkvm && vmware-checkvm >/dev/null - then - # vmware-checkvm is provided by VMware's open-vm-tools - echo vmware - return 0 - elif test -d /proc/xen - then - test -r /proc/xen/capabilities && - if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null - then - # Xen dom0 - return 1 - else - # Xen domU - echo xen - return 0 - fi - fi - return 1 -} - -has_vm_dt() { - # OpenFirmware/Das U-Boot device-tree - test -d /proc/device-tree -} - -check_vm_dt() { - case ${arch} - in - (arm|aarch64) - if test -r /proc/device-tree/hypervisor/compatible - then - if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible - then - echo xen - return 0 - elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible - then - # e.g. VMware ESXi on ARM - echo vmware - return 0 - fi - fi - if glob_exists /proc/device-tree/fw-cfg@*/compatible - then - # qemu,fw-cfg-mmio - sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 - return 0 - fi - if grep -q -F 'dummy-virt' /proc/device-tree/compatible - then - echo lkvm - return 0 - fi - ;; - (ppc64*) - if test -d /proc/device-tree/hypervisor \ - && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible - then - # We are running as a spapr KVM guest on ppc64 - echo kvm-spapr - return 0 - fi - if test -r /proc/device-tree/ibm,partition-name \ - && test -r /proc/device-tree/hmc-managed\? \ - && test -r /proc/device-tree/chosen/qemu,graphic-width - then - echo powervm - fi - ;; - esac - return 1 -} - -has_vm_sys_hypervisor() { - test -d /sys/hypervisor/ -} - -check_vm_sys_hypervisor() { - test -r /sys/hypervisor/type && - case $(head -n1 /sys/hypervisor/type) - in - (xen) - # Ordinary kernel with pv_ops. There does not seem to be - # enough information at present to tell whether this is dom0 - # or domU. - echo xen - return 0 - ;; - esac - return 1 -} - -check_vm_os_specific() { - _hyp_generic=false - - case ${uname_s} - in - (Darwin) - if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ - && test "${hv_vmm_present}" -ne 0 - then - _hyp_generic=true - fi - ;; - (FreeBSD) - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest=$(get_sysctl kern.vm_guest | tolower) && - case ${vm_guest} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${vm_guest} - in - (hv) echo hyperv ;; - (vbox) echo virtualbox ;; - (*) echo "${vm_guest}" ;; - esac - return 0 - ;; - esac - ;; - (NetBSD) - machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && - case ${machdep_hv} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${machdep_hv} - in - (hyper-v) echo hyperv ;; - (xenhvm*) echo xen-hvm ;; - (xenpv*) echo xen-pv ;; - (xen*) echo xen ;; - (*) echo "${machdep_hv}" ;; - esac - return 0 - ;; - esac - ;; - (OpenBSD) - if is_command hostctl && glob_exists /dev/pvbus[0-9]* - then - for _pvbus in /dev/pvbus[0-9]* - do - _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue - case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') - in - (KVM) echo kvm ;; - (Hyper-V) echo hyperv ;; - (VMware) echo vmware ;; - (Xen) echo xen ;; - (bhyve) echo bhyve ;; - (OpenBSD) echo openbsd_vmm ;; - esac - return 0 - done - fi - ;; - (SunOS) - diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') - # NOTE: Don't use -e or -F in Solaris grep - if printf '%s\n' "${diag_conf}" | grep -q -i QEMU - then - echo qemu - return 0 - elif printf '%s\n' "${diag_conf}" | grep -q -i VMware - then - echo vmware - return 0 - fi - ;; - (Linux) - if is_command dmesg - then - while read -r line - do - case ${line} - in - ('Booting paravirtualized kernel on ') - case $(expr "${line}" : '.* kernel on \(.*\)') - in - ('Xen') - echo xen-pv; return 0 ;; - ('bare hardware') - return 1 ;; - esac - ;; - ('Hypervisor detected') - case $(expr "${line}" : '.*: *\(.*\)') - in - ('ACRN') - echo acrn ;; - ('Jailhouse') - echo jailhouse ;; - ('KVM') - echo kvm ;; - ('Microsoft Hyper-V') - echo hyperv ;; - ('VMware') - echo vmware ;; - ('Xen HVM') - echo xen-hvm ;; - ('Xen PV') - echo xen-pv ;; - esac - return 0 - ;; - (lpar:*' under hypervisor') - return 0 ;; - esac - done <<-EOF - $(dmesg 2>/dev/null | awk ' - /Booting paravirtualized kernel on / - /Hypervisor detected: / - /lpar: .* under hypervisor/ - ') - EOF - fi - esac - - # Try to guess hypervisor based on CPU model (sysctl hw.model if available) - if cpu_model=$(get_sysctl hw.model) - then - guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 - fi - - if ${_hyp_generic} - then - # cannot say which hypervisor, but one was detected - return 0 - else - return 1 - fi -} - -run_stage() { - if type "has_$1_$2" >/dev/null 2>&1 - then - "has_$1_$2" - else - true - fi \ - && "check_$1_$2" -} - - -# Execute chroot stages - -for stage in \ - procfs debian_ischroot systemd -do - chrootpnt=$(run_stage chroot ${stage}) || continue - is_chrooted=true - detected_layer 'chroot' - if test -n "${chrootpnt}" - then - echo chroot at "${chrootpnt}" - break - fi -done -if ${is_chrooted:-false} && test -z "${chrootpnt}" -then - # could determine chroot, but not its mount point - echo chroot -fi - - -# Execute container stages - -for stage in \ - systemd pid_1 cgroup files os_specific -do - ctengine=$(run_stage ct ${stage}) || continue - detected_layer 'container' - is_contained=true - if test -n "${ctengine}" - then - echo container on "${ctengine}" - break - fi -done -if ${is_contained:-false} && test -z "${ctengine}" -then - # none of the stages could determine the specific container engine, but - # we are running in some container. - echo container -fi - - -# Execute virtual machine / hypervisor stages - -for stage in \ - systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific -do - hypervisor=$(run_stage vm ${stage}) || continue - detected_layer 'virtual machine' - is_virtual=true - if test -n "${hypervisor}" - then - echo virtual by "${hypervisor}" - break - fi -done -if ${is_virtual:-false} && test -z "${hypervisor}" -then - # none of the stages could determine the specific hypervisor, but - # we are virtual. - echo virtual -fi - - -# Fallback - -detected_layer physical +case "$os" in + "freebsd") + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" + if [ -n "${vm_guest}" ]; then + if [ "${vm_guest}" = "none" ]; then + echo "physical" + exit + fi + echo "virtual_by_${vm_guest}" + exit + fi + ;; + + "openbsd") + # OpenBSD can also use the sysctl's: hw.vendor or hw.product. + # Note we can be reasonably sure about a machine being virtualised + # as long as we can identify the virtualisation technology. + # But not so much about it being physical... + # Patches are welcome / reach out if you have better ideas. + for sysctl in hw.vendor hw.product; do + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" + done + ;; + + *) + # Defaulting to linux for compatibility with previous cdist behaviour + + if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then + echo openvz + exit + fi + + if [ -e "/proc/1/environ" ] && + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then + echo lxc + exit + fi + + if [ -r /proc/cpuinfo ]; then + # this should only exist on virtual guest machines, + # tested on vmware, xen, kvm, bhyve + if grep -q "hypervisor" /proc/cpuinfo; then + # this file is aviable in xen guest systems + if [ -r /sys/hypervisor/type ]; then + if grep -q -i "xen" /sys/hypervisor/type; then + echo virtual_by_xen + exit + fi + else + for vendor_file in /sys/class/dmi/id/product_name \ + /sys/class/dmi/id/sys_vendor \ + /sys/class/dmi/id/chasis_vendor; do + if [ -r ${vendor_file} ]; then + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(cat "${vendor_file}")" + fi + done + fi + echo "virtual_by_unknown" + exit + else + echo "physical" + exit + fi + fi + ;; +esac + +echo "unknown" diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index c6d113cf..63aba9c6 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,18 +27,19 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { printf "%.f\n", ($1 * 1000) } - $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } - $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } - $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } - $2 == "kiB" { printf "%.f\n", ($1 * 1024) } - $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } - $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } - $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' + $2 == "kB" { print $1 * 1000 } + $2 == "MB" { print $1 * 1000 * 1000 } + $2 == "GB" { print $1 * 1000 * 1000 * 1000 } + $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } + $2 == "kiB" { print $1 * 1024 } + $2 == "MiB" { print $1 * 1024 * 1024 } + $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } + $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' } bytes2kib() { - awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' + set -- "$(cat)" + test "$1" -gt 0 && echo $(($1 / 1024)) } diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index b9232ee4..46d87f3e 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,13 +116,6 @@ if [ -f /etc/slackware-version ]; then exit 0 fi -# Appliances - -if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then - echo checkpoint - exit 0 -fi - uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index ec85046f..6489446b 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,9 +34,5 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release -elif test -f /etc/cp-release -then - # Checkpoint firewall or management (actually linux based) - cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 430200ae..3b02dedd 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,7 +1,6 @@ -#!/bin/sh -e +#!/bin/sh # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,22 +17,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# # All os variables are lower case # +# -rc_getvar() { - awk -F= -v varname="$2" ' - function unquote(s) { - if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) - return substr(s, 2, length(s) - 2) - else - return s - } - $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" -} - -case $("${__explorer:?}/os") -in +case "$("$__explorer/os")" in amazon) cat /etc/system-release ;; @@ -41,9 +30,6 @@ in # empty, but well... cat /etc/arch-release ;; - checkpoint) - awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release - ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version @@ -57,8 +43,6 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in - trixie) echo 12.99 ;; - bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; @@ -66,7 +50,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) echo 99.99 ;; + *) exit 1 esac ;; *) @@ -75,23 +59,7 @@ in esac ;; devuan) - devuan_version=$(cat /etc/devuan_version) - case ${devuan_version} - in - (*/ceres) - # ceres versions don't have a number, so we decode by codename: - case ${devuan_version} - in - (chimaera/ceres) echo 3.99 ;; - (beowulf/ceres) echo 2.99 ;; - (ascii/ceres) echo 1.99 ;; - (*) exit 1 - esac - ;; - (*) - echo "${devuan_version}" - ;; - esac + cat /etc/devuan_version ;; fedora) cat /etc/fedora-release @@ -100,20 +68,12 @@ in cat /etc/gentoo-release ;; macosx) - # NOTE: Legacy versions (< 10.3) do not support options - sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' + sw_vers -productVersion ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - if command -v freebsd-version >/dev/null 2>&1 - then - # get userland version - freebsd-version -u - else - # fallback to kernel release for FreeBSD < 10.0 - uname -r - fi + freebsd-version ;; *bsd|solaris) uname -r @@ -138,20 +98,7 @@ in fi ;; ubuntu) - if command -v lsb_release >/dev/null 2>&1 - then - lsb_release -sr - elif test -r /usr/lib/os-release - then - # fallback to /usr/lib/os-release if lsb_release is not present (like - # on minimized Ubuntu installations) - rc_getvar /usr/lib/os-release VERSION_ID - elif test -r /etc/lsb-release - then - # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old - # versions without /usr/lib/os-release. - rc_getvar /etc/lsb-release DISTRIB_RELEASE - fi + lsb_release -sr ;; alpine) cat /etc/alpine-release diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index 6fcd9212..bc47d8de 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,7 +28,6 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 - # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index 8ab268c1..38f1bd3c 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,25 +27,18 @@ else keyid="$__object_id" fi -# From apt-key(8): -# Use of apt-key is deprecated, except for the use of apt-key del in -# maintainer scripts to remove existing keys from the main keyring. -# If such usage of apt-key is desired the additional installation of -# the GNU Privacy Guard suite (packaged in gnupg) is required. -if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then - if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" - then echo present - else echo absent - fi - exit -fi - keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" -if [ -f "$keyfile" ] +if [ -d "$keydir" ] then - echo present - exit + if [ -f "$keyfile" ] + then echo present + else echo absent + fi +else + # fallback to deprecated apt-key + apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ + && echo present \ + || echo absent fi -echo absent diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 17dc9bfc..0c96ff67 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -25,7 +25,11 @@ else fi state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" -method="$(cat "$__object/key_method")" + +if [ "$state_should" = "$state_is" ]; then + # nothing to do + exit 0 +fi keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" @@ -33,18 +37,30 @@ keyfile="$keydir/$__object_id.gpg" case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - # Using __download or __file as key source - # Propagate messages if needed - if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then - if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then - echo "added '$keyid'" >> "$__messages_out" + + if [ -f "$__object/parameter/uri" ]; then + uri="$(cat "$__object/parameter/uri")" + + if [ -d "$keydir" ]; then + cat << EOF + +curl -s -L \\ + -o "$keyfile" \\ + "$uri" + +key="\$( cat "$keyfile" )" + +if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' +then + echo "\$key" | gpg --dearmor > "$keyfile" +fi + +EOF + else + # fallback to deprecated apt-key + echo "curl -s -L '$uri' | apt-key add -" fi - exit 0 - elif [ "${state_is}" = "present" ]; then - exit 0 - fi - # Using key servers to fetch the key - if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then + elif [ -d "$keydir" ]; then # we need to kill gpg after 30 seconds, because gpg # can get stuck if keyserver is not responding. # exporting env var and not exit 1, @@ -84,16 +100,13 @@ EOF echo "added '$keyid'" >> "$__messages_out" ;; absent) - # Removal for keys added from a keyserver without this flag - # is done in the manifest - if [ "$state_is" != "absent" ] && \ - [ -f "$__object/parameter/use-deprecated-apt-key" ]; then + if [ -f "$keyfile" ]; then + echo "rm '$keyfile'" + else # fallback to deprecated apt-key echo "apt-key del \"$keyid\"" - echo "removed '$keyid'" >> "$__messages_out" - # Propagate messages if needed - elif grep -Eq "^__file$keyfile" "$__messages_in"; then - echo "removed '$keyid'" >> "$__messages_out" fi + + echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index e35eaa0f..234bc715 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -10,14 +10,6 @@ DESCRIPTION ----------- Manages the list of keys used by apt to authenticate packages. -This is done by placing the requested key in a file named -``$__object_id.gpg`` in the ``keydir`` directory. - -This is supported by modern releases of Debian-based distributions. - -In order of preference, exactly one of: ``source``, ``uri`` or ``keyid`` -must be specified. - REQUIRED PARAMETERS ------------------- @@ -26,49 +18,21 @@ None. OPTIONAL PARAMETERS ------------------- -keydir - keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is - enabled system-wide by default. - -source - path to a file containing the GPG key of the repository. - Using this is recommended as it ensures that the manifest/type manintainer - has validated the key. - If ``-``, the GPG key is read from the type's stdin. - state 'present' or 'absent'. Defaults to 'present' -uri - the URI from which to download the key. - It is highly recommended that you only use protocols with TLS like HTTPS. - This uses ``__download`` but does not use checksums, if you want to ensure - that the key doesn't change, you are better off downloading it and using - ``--source``. - - -DEPRECATED OPTIONAL PARAMETERS ------------------------------- keyid - the id of the key to download from the ``keyserver``. - This is to be used in absence of ``--source`` and ``--uri`` or together - with ``--use-deprecated-apt-key`` for key removal. - Defaults to ``$__object_id``. + the id of the key to add. Defaults to __object_id keyserver - the keyserver from which to fetch the key. - Defaults to ``pool.sks-keyservers.net``. + the keyserver from which to fetch the key. If omitted the default set + in ./parameter/default/keyserver is used. +keydir + key save location, defaults to ``/etc/apt/trusted.pgp.d`` -DEPRECATED BOOLEAN PARAMETERS ------------------------------ -use-deprecated-apt-key - ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04. - You can use this parameter to force usage of ``apt-key(8)``. - Please only use this parameter to *remove* keys from the keyring, - in order to prepare for removal of ``apt-key``. - Adding keys should be done without this parameter. - This parameter will be removed when Debian 11 stops being supported. +uri + the URI from which to download the key EXAMPLES @@ -76,39 +40,33 @@ EXAMPLES .. code-block:: sh - # add a key that has been verified by a type maintainer - __apt_key jitsi_meet_2021 \ - --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg + # Add Ubuntu Archive Automatic Signing Key + __apt_key 437D05B5 + # Same thing + __apt_key 437D05B5 --state present + # Get rid of it + __apt_key 437D05B5 --state absent - # remove an old, deprecated or expired key - __apt_key jitsi_meet_2016 --state absent + # same thing with human readable name and explicit keyid + __apt_key UbuntuArchiveKey --keyid 437D05B5 - # Get rid of a key that might have been added to - # /etc/apt/trusted.gpg with apt-key - __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent + # same thing with other keyserver + __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com - # add a key that we define in-line - __apt_key jitsi_meet_2021 --source '-' < Ander Punnar -Evilham COPYING ------- -Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can +Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest index 889a764a..010357cd 100755 --- a/cdist/conf/type/__apt_key/manifest +++ b/cdist/conf/type/__apt_key/manifest @@ -2,105 +2,7 @@ __package gnupg -state_should="$(cat "${__object}/parameter/state")" - -incompatible_args() -{ - cat >> /dev/stderr <<-EOF - This type does not support --${1} and --${method} simultaneously. - EOF - exit 1 -} - -if [ -f "${__object}/parameter/source" ]; then - method="source" - src="$(cat "${__object}/parameter/source")" - if [ "${src}" = "-" ]; then - src="${__object}/stdin" - fi -fi -if [ -f "${__object}/parameter/uri" ]; then - if [ -n "${method}" ]; then - incompatible_args uri - fi - method="uri" - src="$(cat "${__object}/parameter/uri")" -fi -if [ -f "${__object}/parameter/keyid" ]; then - if [ -n "${method}" ]; then - incompatible_args keyid - fi - method="keyid" -fi -# Keep old default -if [ -z "${method}" ]; then - method="keyid" -fi -# Save this for later in gencode-remote -echo "${method}" > "${__object}/key_method" - -# Required remotely (most likely already installed) -__package dirmngr -# We need this in case a key has to be dearmor'd -__package gnupg -export require="__package/gnupg" - -if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then - # This is required if apt-key(8) is to be used - if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then - incompatible_args use-deprecated-apt-key - fi -else - if [ "${state_should}" = "absent" ] && \ - [ -f "${__object}/parameter/keyid" ]; then - cat >> /dev/stderr < - - -COPYING -------- -Copyright \(C) 2021 Daniel Fancsali. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest deleted file mode 100755 index 983b2b42..00000000 --- a/cdist/conf/type/__apt_pin/manifest +++ /dev/null @@ -1,68 +0,0 @@ -#!/bin/sh -e -# -# 2021 Daniel Fancsali (fancsali@gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - - -name="$__object_id" - -os=$(cat "$__global/explorer/os") -state="$(cat "$__object/parameter/state")" - -if [ -f "$__object/parameter/package" ]; then - package="$(cat "$__object/parameter/package")" -else - package=$name -fi - -distribution="$(cat "$__object/parameter/distribution")" -priority="$(cat "$__object/parameter/priority")" - - -case "$os" in - debian|ubuntu|devuan) - ;; - *) - printf "This type is specific to Debian and it's derivatives" >&2 - exit 1 - ;; -esac - -case $distribution in - stable|testing|unstable|experimental) - pin="release a=$distribution" - ;; - *) - pin="release n=$distribution" - ;; -esac - - -__file "/etc/apt/preferences.d/$name" \ - --owner root --group root --mode 0644 \ - --state "$state" \ - --source - << EOF -# Created by cdist ${__type##*/} -# Do not change. Changes will be overwritten. -# - -# $name -Package: $package -Pin: $pin -Pin-Priority: $priority -EOF diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority deleted file mode 100644 index 1b79f38e..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/priority +++ /dev/null @@ -1 +0,0 @@ -500 diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional deleted file mode 100644 index 847e703d..00000000 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ /dev/null @@ -1,3 +0,0 @@ -state -package -priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required deleted file mode 100644 index c8572d92..00000000 --- a/cdist/conf/type/__apt_pin/parameter/required +++ /dev/null @@ -1 +0,0 @@ -distribution diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository new file mode 100755 index 00000000..3eb7d491 --- /dev/null +++ b/cdist/conf/type/__apt_ppa/files/remove-apt-repository @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Remove the given apt repository. +# +# Exit with: +# 0: if it worked +# 1: if not +# 2: on other error + +import os +import sys +from aptsources import distro, sourceslist +from softwareproperties import ppa +from softwareproperties.SoftwareProperties import SoftwareProperties + + +def remove_if_empty(file_name): + with open(file_name, 'r') as f: + if f.read().strip(): + return + os.unlink(file_name) + +def remove_repository(repository): + #print 'repository:', repository + codename = distro.get_distro().codename + #print 'codename:', codename + (line, file) = ppa.expand_ppa_line(repository.strip(), codename) + #print 'line:', line + #print 'file:', file + deb_source_entry = sourceslist.SourceEntry(line, file) + src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) + + try: + sp = SoftwareProperties() + sp.remove_source(deb_source_entry) + try: + # If there's a deb-src entry, remove that too + sp.remove_source(src_source_entry) + except: + pass + remove_if_empty(file) + return True + except ValueError: + print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line + return False + +if __name__ == '__main__': + if (len(sys.argv) != 2): + print >> sys.stderr, 'Error: need a repository as argument' + sys.exit(2) + repository = sys.argv[1] + if remove_repository(repository): + sys.exit(0) + else: + sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index e41341b8..84ebebfe 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,9 +29,9 @@ fi case "$state_should" in present) - echo "add-apt-repository -y '$name'" + echo "add-apt-repository '$name'" ;; absent) - echo "add-apt-repository -r -y '$name'" + echo "remove-apt-repository '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index 57e85442..c6f4e876 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,4 +20,9 @@ __package software-properties-common +require="__package/software-properties-common" \ + __file /usr/local/bin/remove-apt-repository \ + --source "$__type/files/remove-apt-repository" \ + --mode 0755 + require="$__object_name" __apt_update_index diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index a28bb45f..d4420e96 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,14 +2,13 @@ set -u entry="$uri $distribution $component" - cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${options} $entry +deb ${forcedarch} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 973b0f6c..1e8592c6 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,21 +22,7 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - -# run 'apt-get update' only if something changed with our sources.list file -# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" + printf 'apt-get update || apt-get update\n' fi diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d317a135..d1acb388 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,9 +23,6 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) -signed-by - provide a GPG key fingerprint or keyring path for signature checks - state 'present' or 'absent', defaults to 'present' @@ -59,11 +56,6 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present - __apt_source goaccess \ - --uri http://deb.goaccess.io/ \ - --component main \ - --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D - AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index cdb526d3..35f15909 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -31,15 +31,9 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - options="arch=$(cat "$__object/parameter/arch")" -fi - -if [ -f "$__object/parameter/signed-by" ]; then - options="$options signed-by=$(cat "$__object/parameter/signed-by")" -fi - -if [ "$options" ]; then - options="[$options]" + forcedarch="[arch=$(cat "$__object/parameter/arch")]" +else + forcedarch="" fi # export variables for use in template @@ -47,7 +41,7 @@ export name export uri export distribution export component -export options +export forcedarch # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 0b5470a1..87537335 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,5 +1,4 @@ state distribution component -arch -signed-by +arch \ No newline at end of file diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 2d7f9030..70b59710 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,23 +18,9 @@ # along with cdist. If not, see . # - -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists -# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get $apt_opts update || apt-get $apt_opts update + apt-get update || apt-get update fi DONE diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state deleted file mode 100644 index f8a3f6c8..00000000 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh -e -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Determine current debconf selections' state. -# Prints one of: -# present: all selections are already set as they should. -# different: one or more of the selections have a different value. -# absent: one or more of the selections are not (currently) defined. -# - -test -x /usr/bin/perl || { - # cannot find perl (no perl ~ no debconf) - echo 'absent' - exit 0 -} - -linesfile="${__object:?}/parameter/line" -test -s "${linesfile}" || { - if test -s "${__object:?}/parameter/file" - then - echo absent - else - echo present - fi - exit 0 -} - -# assert __type_explorer is set (because it is used by the Perl script) -: "${__type_explorer:?}" - -/usr/bin/perl -- - "${linesfile}" <<'EOF' -use strict; -use warnings "all"; - -use Fcntl qw(:DEFAULT :flock); - -use Debconf::Db; -use Debconf::Question; - -# Extract @known... arrays from debconf-set-selections -# These values are required to distinguish flags and values in the given lines. -# DC: I couldn't think of a more ugly solution to the problem… -my @knownflags; -my @knowntypes; -my $debconf_set_selections = '/usr/bin/debconf-set-selections'; -if (-e $debconf_set_selections) { - my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; - eval `sed -n '$sed_known' '$debconf_set_selections'`; -} - -sub mungeline ($) { - my $line = shift; - chomp $line; - $line =~ s/\r$//; - return $line; -} - -sub fatal { printf STDERR @_; exit 1; } - -my $state = 'present'; - -sub state { - my $new = shift; - if ($state eq 'present' - or ($state eq 'different' and $new eq 'absent')) { - $state = $new; - } -} - - -# Load Debconf DB but manually lock on the state explorer script, -# because Debconf aborts immediately if executed concurrently. -# This is not really an ideal solution because the Debconf DB could be locked by -# another process (e.g. apt-get), but no way to achieve this could be found. -# If you know how to, please provide a patch. -my $lockfile = "%ENV{'__type_explorer'}/state"; -if (open my $lock_fh, '+<', $lockfile) { - flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; -} -{ - Debconf::Db->load(readonly => 'true'); -} - - -while (<>) { - # Read and process lines (taken from debconf-set-selections) - $_ = mungeline($_); - while (/\\$/ && ! eof) { - s/\\$//; - $_ .= mungeline(<>); - } - next if /^\s*$/ || /^\s*\#/; - - my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ - or fatal "invalid line: %s\n", $_; - $content = '' unless defined $content; - - - # Compare is and should state - my $q = Debconf::Question->get($label); - - unless (defined $q) { - # probably a preseed - state 'absent'; - next; - } - - if (grep { $_ eq $q->type } @knownflags) { - # This line wants to set a flag, presumably. - if ($q->flag($q->type) ne $content) { - state 'different'; - } - } else { - # Otherwise, it's probably a value… - if ($q->value ne $content) { - state 'different'; - } - - unless (grep { $_ eq $owner } (split /, /, $q->owners)) { - state 'different'; - } - } -} - -printf "%s\n", $state; -EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 9ba28f09..e99aef40 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,37 +17,16 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# +# Setup selections +# -if test -f "${__object:?}/parameter/line" -then - filename="${__object:?}/parameter/line" -elif test -s "${__object:?}/parameter/file" -then - filename=$(cat "${__object:?}/parameter/file") - if test "${filename}" = '-' - then - filename="${__object:?}/stdin" - fi -else - printf 'Neither --line nor --file set.\n' >&2 - exit 1 +filename="$(cat "$__object/parameter/file")" + +if [ "$filename" = "-" ]; then + filename="$__object/stdin" fi -# setting no lines makes no sense -test -s "${filename}" || exit 0 - -state_is=$(cat "${__object:?}/explorer/state") - -if test "${state_is}" != 'present' -then - cat <<-CODE - debconf-set-selections <<'EOF' - $(cat "${filename}") - EOF - CODE - - awk ' - { - printf "set %s %s %s %s\n", $1, $2, $3, $4 - }' "${filename}" >>"${__messages_out:?}" -fi +echo "debconf-set-selections << __file-eof" +cat "$filename" +echo "__file-eof" diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index fd0040ae..58c25b81 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,33 +8,15 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used +On Debian and alike systems debconf-set-selections(1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- -cf. ``--line``. - - -OPTIONAL PARAMETERS -------------------- file - Use the given filename as input for :strong:`debconf-set-selections`\ (1) - If filename is ``-``, read from stdin. - - **This parameter is deprecated, because it doesn't work with state detection.** -line - A line in :strong:`debconf-set-selections`\ (1) compatible format. - This parameter can be used multiple times to set multiple options. - - (This parameter is actually required, but marked optional because the - deprecated ``--file`` is still accepted.) - - -BOOLEAN PARAMETERS ------------------- -None. + Use the given filename as input for debconf-set-selections(1) + If filename is "-", read from stdin. EXAMPLES @@ -42,29 +24,30 @@ EXAMPLES .. code-block:: sh - # Setup gitolite's gituser - __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' + # Setup configuration for nslcd + __debconf_set_selections nslcd --file /path/to/file - # Setup configuration for nslcd from a file. - # NB: Multiple lines can be passed to --line, although this can be considered a hack. - __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" + # Setup configuration for nslcd from another type + __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" + + __debconf_set_selections nslcd --file - << eof + gitolite gitolite/gituser string git + eof SEE ALSO -------- -- :strong:`cdist-type__update_alternatives`\ (7) -- :strong:`debconf-set-selections`\ (1) +:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) AUTHORS ------- -| Nico Schottelius -| Dennis Camera +Nico Schottelius COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file deleted file mode 100644 index 09db545a..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file +++ /dev/null @@ -1 +0,0 @@ -'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple deleted file mode 100644 index a999a0c2..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple +++ /dev/null @@ -1 +0,0 @@ -line diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional b/cdist/conf/type/__debconf_set_selections/parameter/required similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/optional rename to cdist/conf/type/__debconf_set_selections/parameter/required diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index c8f36712..ba7621a1 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,12 +37,6 @@ state source forwarded to :strong:`__file` type -file - forwarded to :strong:`__file` type - This can be used if multiple users need to have a dotfile updated, - which will result in duplicate object id errors. When using the - file parameter the object id can be some unique value. - MESSAGES -------- @@ -67,15 +61,6 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" - # install .vimrc for root and some users - for user in root userx usery userz; do - __dot_file "${user}_dot_vimrc" \ - --user $user \ - --file .vimrc \ - --state exists \ - --source "$__files/$user/.vimrc" - done - SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index a38ed943..02dadf05 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,19 +20,13 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" -if [ -f "${__object}/parameter/file" ]; then - file="$(cat "${__object}/parameter/file")" -else - file="${__object_id}" -fi - # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${file} +subpath=${__object_id} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -70,4 +64,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd new file mode 100755 index 00000000..e3e35b45 --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd @@ -0,0 +1,19 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cmd="$( cat "$__object/parameter/cmd-get" )" + +elif command -v curl > /dev/null +then + cmd="curl -L -o - '%s'" + +elif command -v fetch > /dev/null +then + cmd="fetch -o - '%s'" + +else + cmd="wget -O - '%s'" +fi + +echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get deleted file mode 100755 index 9f1cd59c..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd_get +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cat "$__object/parameter/cmd-get" -elif - command -v curl > /dev/null -then - echo "curl -sSL -o - '%s'" -elif - command -v fetch > /dev/null -then - echo "fetch -o - '%s'" -else - echo "wget -O - '%s'" -fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum deleted file mode 100755 index 84df663c..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd_sum +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh -e - -if [ ! -f "$__object/parameter/sum" ] -then - exit 0 -fi - -if [ -f "$__object/parameter/cmd-sum" ] -then - cat "$__object/parameter/cmd-sum" - exit 0 -fi - -sum_should="$( cat "$__object/parameter/sum" )" - -if echo "$sum_should" | grep -Fq ':' -then - sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" -else - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_hash='cksum' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' - then - sum_hash='md5' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' - then - sum_hash='sha1' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' - then - sum_hash='sha256' - else - echo 'hash format detection failed' >&2 - exit 1 - fi -fi - -os="$( "$__explorer/os" )" - -case "$sum_hash" in - cksum) - echo "cksum %s | awk '{print \$1\" \"\$2}'" - ;; - md5) - case "$os" in - freebsd) - echo "md5 -q %s" - ;; - *) - echo "md5sum %s | awk '{print \$1}'" - ;; - esac - ;; - sha1) - case "$os" in - freebsd) - echo "sha1 -q %s" - ;; - *) - echo "sha1sum %s | awk '{print \$1}'" - ;; - esac - ;; - sha256) - case "$os" in - freebsd) - echo "sha256 -q %s" - ;; - *) - echo "sha256sum %s | awk '{print \$1}'" - ;; - esac - ;; - *) - # we arrive here only if --sum is given with unknown format prefix - echo "unknown hash format: $sum_hash" >&2 - exit 1 - ;; -esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 8c5d5ce1..00362545 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,11 +1,6 @@ #!/bin/sh -e -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi +dst="/$__object_id" if [ ! -f "$dst" ] then @@ -13,27 +8,59 @@ then exit 0 fi -if [ ! -f "$__object/parameter/sum" ] -then - echo 'present' - exit 0 -fi - sum_should="$( cat "$__object/parameter/sum" )" -if echo "$sum_should" | grep -Fq ':' +if [ -f "$__object/parameter/cmd-sum" ] then - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + # shellcheck disable=SC2059 + sum_is="$( eval "$( printf \ + "$( cat "$__object/parameter/cmd-sum" )" \ + "$dst" )" )" +else + os="$( "$__explorer/os" )" + + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" + + elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' + then + case "$os" in + freebsd) + sum_is="md5:$( md5 -q "$dst" )" + ;; + *) + sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' + then + case "$os" in + freebsd) + sum_is="sha1:$( sha1 -q "$dst" )" + ;; + *) + sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' + then + case "$os" in + freebsd) + sum_is="sha256:$( sha256 -q "$dst" )" + ;; + *) + sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" + ;; + esac + fi fi -sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" - -# shellcheck disable=SC2059 -sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" - if [ -z "$sum_is" ] then - echo 'existing destination checksum failed' >&2 + echo 'no checksum from target' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index d1b0d0d5..571d2c3c 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,133 +11,34 @@ fi url="$( cat "$__object/parameter/url" )" -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi +tmp="$( mktemp )" + +dst="/$__object_id" if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + elif command -v curl > /dev/null then - cmd="curl -sSL -o - '%s'" + cmd="curl -L -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - else - echo 'local download failed, no usable utility' >&2 + echo 'no usable locally installed utility for downloading' >&2 exit 1 fi -echo "download_tmp=\"\$( mktemp )\"" - -# shellcheck disable=SC2059 -printf "$cmd > \"\$download_tmp\"\n" "$url" - -if [ -f "$__object/parameter/sum" ] -then - sum_should="$( cat "$__object/parameter/sum" )" - - if [ -f "$__object/parameter/cmd-sum" ] - then - local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" - else - if echo "$sum_should" | grep -Fq ':' - then - sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" - - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" - else - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_hash='cksum' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' - then - sum_hash='md5' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' - then - sum_hash='sha1' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' - then - sum_hash='sha256' - else - echo 'hash format detection failed' >&2 - exit 1 - fi - fi - - case "$sum_hash" in - cksum) - local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" - ;; - md5) - if command -v md5 > /dev/null - then - local_cmd_sum="md5 -q %s" - elif - command -v md5sum > /dev/null - then - local_cmd_sum="md5sum %s | awk '{print \$1}'" - fi - ;; - sha1) - if command -v sha1 > /dev/null - then - local_cmd_sum="sha1 -q %s" - elif - command -v sha1sum > /dev/null - then - local_cmd_sum="sha1sum %s | awk '{print \$1}'" - fi - ;; - sha256) - if command -v sha256 > /dev/null - then - local_cmd_sum="sha256 -q %s" - elif - command -v sha256sum > /dev/null - then - local_cmd_sum="sha256sum %s | awk '{print \$1}'" - fi - ;; - *) - # we arrive here only if --sum is given with unknown format prefix - echo "unknown hash format: $sum_hash" >&2 - exit 1 - ;; - esac - - if [ -z "$local_cmd_sum" ] - then - echo 'local checksum verification failed, no usable utility' >&2 - exit 1 - fi - fi - - # shellcheck disable=SC2059 - echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" - - echo "if [ \"\$sum_is\" != '$sum_should' ]; then" - - echo "echo 'local download checksum mismatch' >&2" - - echo "rm -f \"\$download_tmp\"" - - echo 'exit 1; fi' -fi +printf "$cmd > %s\n" \ + "$url" \ + "$tmp" if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -146,10 +47,12 @@ else target_host="$__target_host" fi -# shellcheck disable=SC2016 -printf '%s "$download_tmp" %s:%s\n' \ +printf '%s %s %s:%s\n' \ "$__remote_copy" \ + "$tmp" \ "$target_host" \ "$dst" -echo "rm -f \"\$download_tmp\"" +echo "rm -f '$tmp'" + +echo 'downloaded' > "$__messages_out" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index e49bcec3..029a0801 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,51 +6,17 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" + cmd="$( cat "$__object/explorer/remote_cmd" )" url="$( cat "$__object/parameter/url" )" - if [ -f "$__object/parameter/destination" ] - then - dst="$( cat "$__object/parameter/destination" )" - else - dst="/$__object_id" - fi + dst="/$__object_id" - echo "download_tmp=\"\$( mktemp )\"" + printf "$cmd > %s\n" \ + "$url" \ + "$dst" - # shellcheck disable=SC2059 - printf "$cmd_get > \"\$download_tmp\"\n" "$url" - - if [ -f "$__object/parameter/sum" ] - then - sum_should="$( cat "$__object/parameter/sum" )" - - if [ -f "$__object/parameter/cmd-sum" ] - then - remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" - else - remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" - - if echo "$sum_should" | grep -Fq ':' - then - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" - fi - fi - - # shellcheck disable=SC2059 - echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" - - echo "if [ \"\$sum_is\" != '$sum_should' ]; then" - - echo "echo 'remote download checksum mismatch' >&2" - - echo "rm -f \"\$download_tmp\"" - - echo 'exit 1; fi' - fi - - echo "mv \"\$download_tmp\" '$dst'" + echo 'downloaded' > "$__messages_out" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index c16510a9..54503470 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,10 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``curl``, ``fetch`` or ``wget``. +Destination (``$__object_id``) in target host must be persistent storage +in order to calculate checksum and decide if file must be (re-)downloaded. + +By default type will try to use ``wget``, ``curl`` or ``fetch``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,40 +19,23 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). -To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. - REQUIRED PARAMETERS ------------------- url File's URL. +sum + Checksum of file going to be downloaded. + By default output of ``cksum`` without filename is expected. + Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + OPTIONAL PARAMETERS ------------------- -destination - Downloaded file's destination in target. If unset, ``$__object_id`` is used. - -sum - Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. - - Type tries to detect hash format with regexes, but prefixes - ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. - - Checksum have two purposes - state check and post-download verification. - In state check, if destination checksum mismatches, then content of URL - will be downloaded to temporary file. If downloaded temporary file's - checksum matches, then it will be moved to destination (overwritten). - - For local downloads it is expected that usable utilities for checksum - calculation exist in the system. - download - If ``local`` (default), then file is downloaded to local storage and copied - to target host. If ``remote``, then download happens in target. - - For local downloads it is expected that usable utilities for downloading - exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. + If ``local`` (default), then download file to local storage and copy + it to target host. If ``remote``, then download happens in target. cmd-get Command used for downloading. @@ -79,7 +65,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum 46da3021ca9eace277115ec9106c5b46 + --sum md5:46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ @@ -95,7 +81,7 @@ Ander Punnar COPYING ------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it +Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 3d4c498b..7ec8d86d 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" +if grep -Eq '^wget' "$__object/explorer/remote_cmd" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index e809ef78..838e2fbf 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,6 +1,4 @@ cmd-get cmd-sum -destination download onchange -sum diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required index 96cdd3b9..6ea4c38f 100644 --- a/cdist/conf/type/__download/parameter/required +++ b/cdist/conf/type/__download/parameter/required @@ -1 +1,2 @@ url +sum diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 5a303308..231b6927 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -72,7 +72,6 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 - echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -89,39 +88,27 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" - if [ "$create_file" ]; then - # When creating an empty file we create it locally and then - # upload it so that permissions can be set before moving the file - # into place. - source="$__object/files/empty" - touch "$source" - fi - # upload file to temp location - upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" - # Yes, we are aware that this is a race condition. - # However: - # a) cdist usually writes to directories that are not user writable - # (probably > 99.9%) - # b) if they are user owned, the user / attacker always wins - # (probably < 0.1%) - # c) the only case which we could improve are tmp directories and we - # don't think managing tmp directories with cdist is a typical case - # ("the rest %)" - - # Tell gencode-remote to where we uploaded the file so it can move - # it to its final destination. - echo "$upload_destination" > "$__object/files/upload-destination" - - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi + tempfile_template="${destination}.cdist.XXXXXXXXXX" cat << DONE -$__remote_copy "$source" "${my_target_host}:${upload_destination}" +destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +DONE + if [ "$upload_file" ]; then + echo upload >> "$__messages_out" + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE +$__remote_copy "$source" "${my_target_host}:\$destination_upload" +DONE + fi +# move uploaded file into place +cat << DONE +$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 1a9ff69c..f7a528fd 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,13 +62,6 @@ set_mode() { case "$state_should" in present|exists) - if [ -f "$__object/files/upload-destination" ]; then - final_destination="$destination" - # We change the 'global' $destination variable here so we can - # change attributes of the new/uploaded file before moving it - # to it's final destination. - destination="$(cat "$__object/files/upload-destination")" - fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -88,11 +81,6 @@ case "$state_should" in fi fi done - if [ -f "$__object/files/upload-destination" ]; then - # move uploaded file into place - printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv "%s" "%s"\n' "$destination" "$final_destination" - fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index d376c09f..9be3c575 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|gentoo|redhat|suse|ubuntu) + alpine|centos|fedora|redhat|suse|gentoo) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index ab4396b1..3ddf9656 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -1,24 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}/.git" +destination="/$__object_id/.git" -# shellcheck disable=SC2012 -group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((group_gid + 1)) -ge 0 -then - group_should=$(cat "${__object:?}/parameter/group") - - if expr "${group_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${group_gid}" - else - if command -v getent > /dev/null - then - getent group "${group_gid}" | cut -d : -f 1 - else - awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group - fi - fi -fi +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 4a4d0d13..4c3cd431 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -1,19 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}/.git" +destination="/$__object_id/.git" -# shellcheck disable=SC2012 -owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((owner_uid + 1)) -ge 0 -then - owner_should=$(cat "${__object:?}/parameter/owner") - - if expr "${owner_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${owner_uid}" - else - printf '%s\n' "$(id -u -n "${owner_uid}")" - fi -fi +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 0d944482..d145c4c3 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*|11*) + 10*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http deleted file mode 100644 index 0508a465..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/http +++ /dev/null @@ -1,8 +0,0 @@ -frontend http - bind BIND@:80 - mode http - option httplog - default_backend http - -backend http - mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https deleted file mode 100644 index 73deac46..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/https +++ /dev/null @@ -1,10 +0,0 @@ -frontend https - bind BIND@:443 - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend https - -backend https - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps deleted file mode 100644 index b1ec3793..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/imaps +++ /dev/null @@ -1,12 +0,0 @@ -frontend imaps - bind BIND@:143 - bind BIND@:993 - - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend imaps - -backend imaps - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps deleted file mode 100644 index dce6ed4a..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/smtps +++ /dev/null @@ -1,12 +0,0 @@ -frontend smtps - bind BIND@:25 - bind BIND@:465 - - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend smtps - -backend smtps - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst deleted file mode 100644 index 6c131cbe..00000000 --- a/cdist/conf/type/__haproxy_dualstack/man.rst +++ /dev/null @@ -1,121 +0,0 @@ -cdist-type__haproxy_dualstack(7) -================================ - - -NAME ----- -cdist-type__haproxy_dualstack - Proxy services from a dual-stack server - - -DESCRIPTION ------------ -This (singleton) type installs and configures haproxy to act as a dual-stack -proxy for single-stack services. - -This can be useful to add IPv4 support to IPv6-only services while only using -one IPv4 for many such services. - -By default this type uses the plain TCP proxy mode, which means that there is no -need for TLS termination on this host when SNI is supported. -This also means that proxied services will not receive the client's IP address, -but will see the proxy's IP address instead (that of `$__target_host`). - -This can be solved by using the PROXY protocol, but do take into account that, -e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same -port, so you will need to use other ports for that. - -As a recommendation in this type: use TCP ports 8080 and 591 respectively to -serve HTTP and HTTPS using the PROXY protocol. - -See the EXAMPLES for more details. - - -OPTIONAL PARAMETERS -------------------- -v4proxy - Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. - In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is - the IP address actually providing the proxied services. - The full format of this argument is: - `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` - Where starting with `proxy:` determines that the PROXY protocol must be - used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT - override for the given PROTOCOL (see `--protocol`), if not present the - PROTOCOL's default port will be used. - - -v6proxy - Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. - In its simplest use, it must be a NAME with an `A` DNS entry, which is - the IP address actually providing the proxied services. - See `--v4proxy` for more options and details. - -protocol - Can be passed multiple times or as a space-separated list of protocols. - Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. - This defaults to: `http https imaps smtps`. - - -EXAMPLES --------- - -.. code-block:: sh - - # Proxy the IPv6-only services so IPv4-only clients can access them - # This uses HAProxy's TCP mode for http, https, imaps and smtps - __haproxy_dualstack \ - --v4proxy ipv6.chat \ - --v4proxy matrix.ungleich.ch - - # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them - # Note this means that the backend IPv6-only server will only see - # the IPv6 address of the haproxy host managed by cdist, which can be - # troublesome if this information is relevant for analytics/security/... - # See the PROXY example below - __haproxy_dualstack \ - --protocol http --protocol https \ - --v4proxy ipv6.chat \ - --v4proxy matrix.ungleich.ch - - # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling - # IPv4-only clients to access them while maintaining the client's IP address - __haproxy_dualstack \ - --protocol http --protocol https \ - --v4proxy proxy:ipv6.chat:http=8080:https=591 \ - --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 - # Note however that the PROXY protocol is not compatible with regular - # HTTP(S) protocols, so your nginx will have to listen on different ports - # with the PROXY settings. - # Note that you will need to restrict access to the 8080 port to prevent - # Client IP spoofing. - # This can be something like: - # server { - # # listen for regular HTTP connections - # listen [::]:80 default_server; - # listen 80 default_server; - # # listen for PROXY HTTP connections - # listen [::]:8080 proxy_protocol; - # # Accept the Client's IP from the PROXY protocol - # real_ip_header proxy_protocol; - # } - - -SEE ALSO --------- -- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ -- https://www.haproxy.com/blog/haproxy/proxy-protocol/ -- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ - - -AUTHORS -------- -ungleich -Evilham - - -COPYING -------- -Copyright \(C) 2021 ungleich glarus ag. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest deleted file mode 100644 index d110eea6..00000000 --- a/cdist/conf/type/__haproxy_dualstack/manifest +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/sh -eu - -__package haproxy -require="__package/haproxy" __start_on_boot haproxy - -tmpdir="$__object/files" -mkdir "$tmpdir" -configtmp="$__object/files/haproxy.cfg" - -os=$(cat "$__global/explorer/os") -case $os in - freebsd) - CONFIG_FILE="/usr/local/etc/haproxy.conf" - cat < "$configtmp" -global - maxconn 4000 - user nobody - group nogroup - daemon - -EOF - - ;; - *) - CONFIG_FILE="/etc/haproxy/haproxy.cfg" - cat < "$configtmp" -global - log [::1] local2 - chroot /var/lib/haproxy - pidfile /var/run/haproxy.pid - maxconn 4000 - user haproxy - group haproxy - daemon - - # turn on stats unix socket - stats socket /var/lib/haproxy/stats - -EOF - ;; -esac - -cat <> "$configtmp" -defaults - retries 3 - log global - timeout http-request 10s - timeout queue 1m - timeout connect 10s - timeout client 1m - timeout server 1m - timeout http-keep-alive 10s - timeout check 10s -EOF - -dig_cmd="$(command -v dig || true)" -get_ip() { - # Usage: get_ip (ipv4|ipv6) NAME - # uses "dig" if available, else fallback to "host" - case $1 in - ipv4) - if [ -n "${dig_cmd}" ]; then - ${dig_cmd} +short A "$2" - else - host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' - fi - ;; - ipv6) - if [ -n "${dig_cmd}" ]; then - ${dig_cmd} +short AAAA "$2" - else - host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' - fi - ;; - esac -} - -PROTOCOLS="$(cat "$__object/parameter/protocol")" - -for proxy in v4proxy v6proxy; do - param=$__object/parameter/$proxy - # no backend? skip generating code - if [ ! -f "$param" ]; then - continue - fi - - # turn backend name into bind parameter: v4backend -> ipv4@ - bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') - - case $bind in - ipv4) - backendproto=ipv6 - ;; - ipv6) - backendproto=ipv4 - ;; - esac - - for proto in ${PROTOCOLS}; do - # Add protocol "header" - printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" - - sed -e "s/BIND/$bind/" \ - -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ - -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ - "$__type/files/$proto" >> "$configtmp" - - while read -r hostdefinition; do - if echo "$hostdefinition" | grep -qE '^proxy:'; then - # Proxy protocol was requested - host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" - send_proxy=" send-proxy" - else - # Just use tcp proxy mode - host="$hostdefinition" - send_proxy="" - fi - if echo "$hostdefinition" | grep -qE ":${proto}="; then - # Use custom port definition if requested - port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" - else - # Else use the default - port="" - fi - servername=$host - - res=$(get_ip "$bind" "$servername") - - if [ -z "$res" ]; then - echo "$servername does not resolve - aborting config" >&2 - exit 1 - fi - - # Treat protocols without TLS+SNI specially - if [ "$proto" = http ]; then - echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" - else - echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" - fi - - # Create the "server" itself. - # Note that port and send_proxy will be empty unless - # they were requested by the type user - echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" - - done < "$param" - done -done - -# Create config file -require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 - -require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ - --pattern "^__file${CONFIG_FILE}" \ - --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol deleted file mode 100644 index dc8bb7bf..00000000 --- a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol +++ /dev/null @@ -1 +0,0 @@ -http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple deleted file mode 100644 index 8c482bd4..00000000 --- a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple +++ /dev/null @@ -1,3 +0,0 @@ -protocol -v4proxy -v6proxy diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path new file mode 100755 index 00000000..3c6076df --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path @@ -0,0 +1,3 @@ +#!/bin/sh -e + +command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data deleted file mode 100755 index ff62e742..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh -e -certbot_path="$(command -v certbot 2>/dev/null || true)" -# Defaults -certificate_exists="no" -certificate_is_test="no" - -if [ -n "${certbot_path}" ]; then - # Find python executable that has access to certbot's module - python_path=$(sed -n '1s/^#! *//p' "${certbot_path}") - - # Use a lock for cdist due to certbot not exiting with failure - # or having any flags for concurrent use. - _certbot() { - ${python_path} - 2>/dev/null < "${existing_domains}" - certificate_is_test="$(_explorer_var certificate_is_test)" + existing_domains="${__object}/explorer/certificate-domains" + certificate_is_test=$(cat "${__object}/explorer/certificate-is-test") sort -uo "${requested_domains}" "${requested_domains}" sort -uo "${existing_domains}" "${existing_domains}" diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 638a99e0..1df3574a 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,6 +1,6 @@ #!/bin/sh -certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)" +certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" state=$(cat "${__object}/parameter/state") os="$(cat "${__global:?}/explorer/os")" @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*|11*) + 10*) __package_apt certbot ;; diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index 79c0d9d3..fbfca330 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,24 +81,12 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get $apt_opts update > /dev/null 2>&1 || true +then echo apt-get update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index ca9aa45a..3f88f6bc 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,7 +37,6 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 - # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index a10c16d3..803468b5 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,19 +41,7 @@ fi case "$type" in yum) ;; apt) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - - echo "apt-get --quiet $apt_opts update" + echo "apt-get --quiet update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index d332e851..38aa001e 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,10 +28,6 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" -if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then - apt_with_new_pkgs="--with-new-pkgs" -fi - if [ -f "$type" ]; then type="$(cat "$type")" else @@ -58,7 +54,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget $apt_with_new_pkgs upgrade" + else echo "$aptget upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index 0c116bac..e9e2b8ce 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,14 +33,6 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. -apt-with-new-pkg - Allow installing new packages when used in conjunction with - upgrade. This is useful if the update of an installed package - requires new dependencies to be installed. Instead of holding the - package back upgrade will upgrade the package and install the new - dependencies. Note that upgrade with this option will never remove - packages, only allow adding new ones. - apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index cd22eb90..7a56a34b 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,3 +1,2 @@ apt-clean apt-dist-upgrade -apt-with-new-pkgs diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user deleted file mode 100644 index c6582dc4..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/postgres_user +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -os=$("${__explorer:?}/os") - -case ${os} -in - (alpine) - echo 'postgres' - ;; - (centos|rhel|scientific) - echo 'postgres' - ;; - (debian|devuan|ubuntu) - echo 'postgres' - ;; - (freebsd) - test -x /usr/local/etc/rc.d/postgresql || { - printf 'could not find postgresql rc script./n' >&2 - exit 1 - } - pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || { - printf 'postgresql daemon is not running.\n' >&2 - exit 1 - } - pg_pid=$(printf '%s\n' "${pg_status}" \ - | sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p') - - # PostgreSQL < 9.6: pgsql - # PostgreSQL >= 9.6: postgres - ps -o user -p "${pg_pid}" | sed -n '2p' - ;; - (netbsd) - echo 'pgsql' - ;; - (openbsd) - echo '_postgresql' - ;; - (suse) - echo 'postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state deleted file mode 100644 index 4b7b0a43..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ /dev/null @@ -1,223 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -postgres_user=$("${__type_explorer:?}/postgres_user") -conf_name=${__object_id:?} - -tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; } - -tobytes() { - # NOTE: This function treats everything as base 2. - # It is not compatible with SI units. - awk 'BEGIN { FS = "\n" } - /TB$/ { $0 = ($0 * 1024) "GB" } - /GB$/ { $0 = ($0 * 1024) "MB" } - /MB$/ { $0 = ($0 * 1024) "kB" } - /kB$/ { $0 = ($0 * 1024) "B" } - /B?$/ { sub(/ *B?$/, "") } - ($0*1) == $0 # is number - ' <<-EOF - $1 - EOF -} - -tomillisecs() { - awk 'BEGIN { FS = "\n" } - /d$/ { $0 = ($0 * 24) "h" } - /h$/ { $0 = ($0 * 60) "min" } - /min$/ { $0 = ($0 * 60) "s" } - /[^m]s$/ { $0 = ($0 * 1000) "ms" } - /ms$/ { $0 *= 1 } - ($0*1) == $0 # is number - ' <<-EOF - $1 - EOF -} - -tobool() { - # prints either 'on' or 'off' - case $(tolower "$1") - in - (t|true|y|yes|on|1) - echo 'on' ;; - (f|false|n|no|off|0) - echo 'off' ;; - (*) - printf 'Inavlid bool value: %s\n' "$2" >&2 - return 1 - ;; - esac - return 0 -} - -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } -psql_exec() { - su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" -} - -psql_conf_source() { - # NOTE: SHOW/SET are case-insentitive, so this command should also be. - psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" -} -psql_conf_cmp() ( - IFS='|' read -r lower_name vartype setting unit <<-EOF - $(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')") - EOF - - should_value=$2 - is_value=${setting} - - # The following case contains special cases for special settings. - case ${lower_name} - in - (archive_command) - if test "${setting}" = '(disabled)' - then - # DAFUQ PostgreSQL?! - # PostgreSQL returns (disabled) if the feature is inactive. - # We cannot compare the values unless it is enabled, first. - return 0 - fi - ;; - (archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit) - # Although only 'on', 'off' are documented, PostgreSQL accepts all - # the "likely" variants of "on" and "off". - case $(tolower "${should_value}") - in - (on|off|true|false|yes|no|1|0) - should_value=$(tobool "${should_value}") - ;; - esac - ;; - esac - - case ${vartype} - in - (bool) - test -z "${unit}" || { - # please fix the explorer if this error occurs. - printf 'units are not supported for vartype: %s\n' "${vartype}" >&2 - exit 1 - } - - should_value=$(tobool "${should_value}") - - test "${is_value}" = "${should_value}" - ;; - (enum) - test -z "${unit}" || { - # please fix the explorer if this error occurs. - printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 - exit 1 - } - - # NOTE: All enums that are currently defined are lower case, but - # PostgreSQL also accepts upper case spelling. - should_value=$(tolower "$2") - - test "${is_value}" = "${should_value}" - ;; - (integer) - # split multiples from unit, first (e.g. 8kB -> 8, kB) - case ${unit} - in - ([0-9]*) - multiple=${unit%%[!0-9]*} - unit=${unit##*[0-9 ]} - ;; - (*) multiple=1 ;; - esac - - is_value=$((setting * multiple))${unit} - - if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null - then - # default unit - should_value=$((should_value * multiple))${unit} - fi - - # then, do conversion - # NOTE: these conversions work for integers only! - case ${unit} - in - (B|[kMGT]B) - # bytes - is_bytes=$(tobytes "${is_value}") - should_bytes=$(tobytes "${should_value}") - - test $((is_bytes)) -eq $((should_bytes)) - ;; - (ms|s|min|h|d) - # seconds - is_ms=$(tomillisecs "${is_value}") - should_ms=$(tomillisecs "${should_value}") - - test $((is_ms)) -eq $((should_ms)) - ;; - ('') - # no unit - is_int=${is_value} - should_int=${should_value} - - test $((is_int)) -eq $((should_int)) - ;; - esac - ;; - (real|string) - # NOTE: reals could possibly have units, but currently there none. - - test -z "${unit}" || { - # please fix the explorer if this error occurs. - printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 - exit 1 - } - - test "${is_value}" = "${should_value}" - ;; - esac -) - -psql_exec 'SELECT 1' >/dev/null || { - echo 'Connection to PostgreSQL server failed' >&2 - exit 1 -} - -case $(psql_conf_source "${conf_name}") -in - ('') - printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2 - exit 1 - ;; - (default) - echo absent - ;; - (*) - if ! test -f "${__object:?}/parameter/value" - then - echo present - elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" - then - echo present - else - echo different - fi - ;; -esac diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote deleted file mode 100755 index 27651600..00000000 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -state_is=$(cat "${__object:?}/explorer/state") -state_should=$(cat "${__object:?}/parameter/state") -postgres_user=$(cat "${__object:?}/explorer/postgres_user") - -conf_name=${__object_id:?} - -if test "${state_is}" = "${state_should}" -then - exit 0 -fi - -quote() { - for _arg - do - shift - if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" - then - # needs quoting - set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" - else - set -- "$@" "${_arg}" - fi - done - unset _arg - - # NOTE: Use printf because POSIX echo interprets escape sequences - printf '%s' "$*" -} - - -psql_cmd() { - printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" -} - -case ${state_should} -in - (present) - test -n "${__object:?}/parameter/value" || { - echo 'Missing required parameter --value' >&2 - exit 1 - } - - cat <<-EOF - exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL' - \\set conf_value \`cat <&3\` - ALTER SYSTEM SET ${conf_name} = :'conf_value'; - SELECT pg_reload_conf(); - SQL - exec 3<&- - EOF - ;; - (absent) - psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT" - ;; - (*) - printf 'Invalid --state: %s\n' "${state_should}" >&2 - printf 'Only "present" and "absent" are acceptable.\n' >&2 - exit 1 - ;; -esac - -# Restart PostgreSQL server if required to apply new configuration value -cat <&2 - exit 1 - esac - ;; - (*) - printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2 - exit 1 - esac - ) -fi -EOF diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst deleted file mode 100644 index e035f080..00000000 --- a/cdist/conf/type/__postgres_conf/man.rst +++ /dev/null @@ -1,60 +0,0 @@ -cdist-type__postgres_conf(7) -============================ - -NAME ----- -cdist-type__postgres_conf - Alter PostgreSQL configuration - - -DESCRIPTION ------------ -Configure a running PostgreSQL server using ``ALTER SYSTEM``. - - -REQUIRED PARAMETERS -------------------- -value - The value to set (can be omitted if ``--state`` is set to ``absent``). - - -OPTIONAL PARAMETERS -------------------- -state - ``present`` or ``absent``. - Defaults to ``present``. - - -BOOLEAN PARAMETERS ------------------- -None. - - -EXAMPLES --------- - -.. code-block:: sh - - # set timezone - __postgres_conf timezone --value Europe/Zurich - - # reset maximum number of concurrent connections to default (normally 100) - __postgres_conf max_connections --state absent - - -SEE ALSO --------- -None. - - -AUTHORS -------- -Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch) -Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch) - - -COPYING -------- -Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch). -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__postgres_conf/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional deleted file mode 100644 index d0460d86..00000000 --- a/cdist/conf/type/__postgres_conf/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -state -value diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user deleted file mode 120000 index 714e7237..00000000 --- a/cdist/conf/type/__postgres_database/explorer/postgres_user +++ /dev/null @@ -1 +0,0 @@ -../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index 6a25df86..d68d4120 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -1,7 +1,6 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,18 +18,25 @@ # along with cdist. If not, see . # -postgres_user=$("${__type_explorer:?}/postgres_user") +case "$("${__explorer}/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac -dbname=${__object_id:?} -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } -psql_exec() { - su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")" -} +name="$__object_id" -if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}" +if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 7d7d6fa2..0f11cff4 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,63 +18,60 @@ # along with cdist. If not, see . # -quote() { - for _arg - do - shift - if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" - then - # needs quoting - set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" - else - set -- "$@" "${_arg}" - fi - done - unset _arg - - # NOTE: Use printf because POSIX echo interprets escape sequences - printf '%s' "$*" -} - -postgres_user=$(cat "${__object:?}/explorer/postgres_user") - -dbname=${__object_id:?} -state_should=$(cat "${__object:?}/parameter/state") -state_is=$(cat "${__object:?}/explorer/state") - -if test "${state_should}" = "$state_is" -then - exit 0 -fi - -case ${state_should} +case "$(cat "${__global}/explorer/os")" in - (present) - set -- - - while read -r param_name opt - do - if test -f "${__object:?}/parameter/${param_name}" - then - set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")" - fi - done <<-'EOF' - owner -O - template --template - encoding --encoding - lc_collate --lc-collate - lc_ctype --lc-ctype - EOF - - set -- "$@" "${dbname}" - - cat <<-EOF - su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")") - EOF - ;; - (absent) - cat <<-EOF - su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")") - EOF - ;; + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; esac + + +name="$__object_id" +state_should="$(cat "$__object/parameter/state")" +state_is="$(cat "$__object/explorer/state")" + +if [ "$state_should" != "$state_is" ]; then + case "$state_should" in + present) + owner="" + if [ -f "$__object/parameter/owner" ]; then + owner="-O \"$(cat "$__object/parameter/owner")\"" + fi + + template="" + if [ -f "$__object/parameter/template" ]; then + template="--template \"$(cat "$__object/parameter/template")\"" + fi + + encoding="" + if [ -f "$__object/parameter/encoding" ]; then + encoding="--encoding \"$(cat "$__object/parameter/encoding")\"" + fi + + lc_collate="" + if [ -f "$__object/parameter/lc-collate" ]; then + lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\"" + fi + + lc_ctype="" + if [ -f "$__object/parameter/lc-ctype" ]; then + lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\"" + fi + + cat << EOF +su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype" +EOF + ;; + absent) + cat << EOF +su - '$postgres_user' -c "dropdb \"$name\"" +EOF + ;; + esac +fi diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user deleted file mode 120000 index 714e7237..00000000 --- a/cdist/conf/type/__postgres_extension/explorer/postgres_user +++ /dev/null @@ -1 +0,0 @@ -../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state deleted file mode 100644 index 9d156be7..00000000 --- a/cdist/conf/type/__postgres_extension/explorer/state +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Prints "present" if the extension is currently installed. -# "absent" otherwise. - -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } - -postgres_user=$("${__type_explorer:?}/postgres_user") - -IFS=: read -r dbname extname <. # -postgres_user=$(cat "${__object:?}/explorer/postgres_user") - -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } -psql_cmd() { - printf 'su - %s -c %s\n' \ - "$(quote "${postgres_user}")" \ - "$(quote psql "$(quote "$1")" -c "$(quote "$2")")" -} - - -IFS=: read -r dbname extname <&2 - exit 1 - ;; + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + +dbname=$( echo "$__object_id" | cut -d":" -f1 ) +extension=$( echo "$__object_id" | cut -d":" -f2 ) + +state_should=$( cat "$__object/parameter/state" ) + +case "$state_should" in + present) + cmd="CREATE EXTENSION IF NOT EXISTS $extension" + echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'" + ;; + absent) + cmd="DROP EXTENSION IF EXISTS $extension" + echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'" + ;; esac diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 442239f6..79645b2b 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -3,36 +3,32 @@ cdist-type__postgres_extension(7) NAME ---- -cdist-type__postgres_extension - Manage PostgreSQL extensions +cdist-type__postgres_extension - manage postgres extensions DESCRIPTION ----------- -This cdist type allows you to manage PostgreSQL extensions. +This cdist type allows you to create or drop postgres extensions. -The ``__object_id`` to pass to ``__postgres_extension`` is of the form -``dbname:extension``, e.g.: +The object you need to pass to __postgres_extension consists of +the database name and the extension name joined by a colon in the +following form: + +.. code-block:: sh + + dbname:extension + +f.ex. .. code-block:: sh rails_test:unaccent -**CAUTION!** Be careful when installing extensions from (untrusted) third-party -sources: - - | Installing an extension as superuser requires trusting that the extension's - author wrote the extension installation script in a secure fashion. It is - not terribly difficult for a malicious user to create trojan-horse objects - that will compromise later execution of a carelessly-written extension - script, allowing that user to acquire superuser privileges. - | – ``_ - - OPTIONAL PARAMETERS ------------------- state - either ``present`` or ``absent``, defaults to ``present``. + either "present" or "absent", defaults to "present" EXAMPLES @@ -40,29 +36,24 @@ EXAMPLES .. code-block:: sh - # Install extension unaccent into database rails_test - __postgres_extension rails_test:unaccent - - # Drop extension unaccent from database fails_test - __postgres_extension rails_test:unaccent --state absent + __postgres_extension rails_test:unaccent + __postgres_extension --present rails_test:unaccent + __postgres_extension --absent rails_test:unaccent SEE ALSO -------- -- :strong:`cdist-type__postgres_database`\ (7) -- PostgreSQL "CREATE EXTENSION" documentation at: - ``_. +:strong:`cdist-type__postgre_database`\ (7) +Postgres "Create Extension" documentation at: . -AUTHORS +AUTHOR ------- -| Tomas Pospisek -| Dennis Camera - +Tomas Pospisek COPYING ------- -Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera. -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. +Copyright \(C) 2014 Tomas Pospisek. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user deleted file mode 120000 index 714e7237..00000000 --- a/cdist/conf/type/__postgres_role/explorer/postgres_user +++ /dev/null @@ -1 +0,0 @@ -../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 822816c1..34069de9 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -19,7 +19,19 @@ # along with cdist. If not, see . # -postgres_user=$("${__type_explorer:?}/postgres_user") +case $("${__explorer:?}/os") +in + (netbsd) + postgres_user='pgsql' + ;; + (openbsd) + postgres_user='_postgresql' + ;; + (*) + postgres_user='postgres' + ;; +esac + rolename=${__object_id:?} @@ -43,7 +55,8 @@ role_properties=$( BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } - NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }' + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } + ' ) if test -n "${role_properties}" @@ -77,10 +90,12 @@ then # Check password passwd_stored=$( psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ - | awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }') - passwd_stored=${passwd_stored%.} + | awk 'BEGIN { RS = "\036" } NR == 2' + printf . + ) + passwd_stored=${passwd_stored%?.} - if test -s "${__object:?}/parameter/password" + if test -f "${__object:?}/parameter/password" then passwd_should=$(cat "${__object:?}/parameter/password"; printf .) fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 4cb78330..d7631fbd 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -28,7 +28,20 @@ quote() { fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" } -postgres_user=$(cat "${__object:?}/explorer/postgres_user") +case $(cat "${__global:?}/explorer/os") +in + (netbsd) + postgres_user='pgsql' + ;; + (openbsd) + postgres_user='_postgresql' + ;; + (*) + postgres_user='postgres' + ;; +esac + + rolename=${__object_id:?} state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") @@ -46,7 +59,7 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! - # NOTE: Never set an empty string as the password, because it can be + # NOTE: Never set an empty string as the password, because they can be # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index 922ce3df..a655bda7 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -1,24 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}" +destination="/$__object_id" -# shellcheck disable=SC2012 -group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((group_gid + 1)) -ge 0 -then - group_should=$(cat "${__object:?}/parameter/group") - - if expr "${group_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${group_gid}" - else - if command -v getent >/dev/null 2>&1 - then - getent group "${group_gid}" | cut -d : -f 1 - else - awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group - fi - fi -fi +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index ebec751f..8b3c7f8e 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -1,19 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}" +destination="/$__object_id" -# shellcheck disable=SC2012 -owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((owner_uid + 1)) -ge 0 -then - owner_should=$(cat "${__object:?}/parameter/owner") - - if expr "${owner_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${owner_uid}" - else - printf '%s\n' "$(id -u -n "${owner_uid}")" - fi -fi +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e9f3c131..e36ded2f 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,104 +1,39 @@ #!/bin/sh -e - -if ! command -v rsync > /dev/null -then - echo 'rsync is missing in local machine' >&2 - exit 1 -fi - -src="$( cat "$__object/parameter/source" )" - -if [ ! -e "$src" ] -then - echo "$src not found" >&2 - exit 1 -fi - -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi - -# if source is directory, then make sure that -# source and destination are ending with slash, -# because this is what you almost always want when -# rsyncing two directories. - -if [ -d "$src" ] -then - if ! echo "$src" | grep -Eq '/$' - then - src="$src/" - fi - - if ! echo "$dst" | grep -Eq '/$' - then - dst="$dst/" - fi -fi - -remote_user="$( cat "$__object/parameter/remote-user" )" - -options="$( cat "$__object/parameter/options" )" - -if [ -f "$__object/parameter/option" ] -then - while read -r l - do - # there's a limitation in argparse: value can't begin with '-'. - # to workaround this, let's prefix opts with '\' in manifest and remove here. - # read more about argparse issue: https://bugs.python.org/issue9334 - - options="$options $( echo "$l" | sed 's/\\//g' )" - done \ - < "$__object/parameter/option" -fi - -if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] -then - options="$options --chown=" - - if [ -f "$__object/parameter/owner" ] - then - owner="$( cat "$__object/parameter/owner" )" - options="$options$owner" - fi - - if [ -f "$__object/parameter/group" ] - then - group="$( cat "$__object/parameter/group" )" - options="$options:$group" - fi -fi - -if [ -f "$__object/parameter/mode" ] -then - mode="$( cat "$__object/parameter/mode" )" - options="$options --chmod=$mode" -fi - -# IMPORTANT # -# 1. we first dry-run rsync with change summary to find out -# if there are any changes and code generation is needed. -# 2. normally, to get current state or target host, we run -# such operations in type explorers, but that's not -# possible due to how rsync works. -# 3. redirecting output of dry-run to stderr to ease debugging. -# 4. to understand how that cryptic regex works, please -# open rsync manpage and read about --itemize-changes. +# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# -export RSYNC_RSH="$__remote_exec" +source=$(cat "$__object/parameter/source") +remote_user=$(cat "$__object/parameter/remote-user") -# shellcheck disable=SC2086 -if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ - | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 -then - exit 0 +if [ -f "$__object/parameter/destination" ]; then + destination=$(cat "$__object/parameter/destination") +else + destination="/$__object_id" fi -echo "export RSYNC_RSH='$__remote_exec'" +set -- +if [ -f "$__object/parameter/rsync-opts" ]; then + while read -r opts; do + set -- "$@" "--$opts" + done < "$__object/parameter/rsync-opts" +fi -echo "rsync $options $src $remote_user@$__target_host:$dst" +echo rsync -a \ + --no-owner --no-group \ + -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__rsync/gencode-remote similarity index 56% rename from cdist/conf/type/__debconf_set_selections/manifest rename to cdist/conf/type/__rsync/gencode-remote index 0f4fb2e2..074246af 100755 --- a/cdist/conf/type/__debconf_set_selections/manifest +++ b/cdist/conf/type/__rsync/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2015 Dominique Roux (dominique.roux4 at gmail.com) # # This file is part of cdist. # @@ -18,4 +18,20 @@ # along with cdist. If not, see . # -__package_apt debconf +if [ -f "$__object/parameter/destination" ]; then + destination=$(cat "$__object/parameter/destination") +else + destination="/$__object_id" +fi + +ownergroup="" +if [ -f "$__object/parameter/owner" ]; then + ownergroup=$(cat "$__object/parameter/owner") +fi +if [ -f "$__object/parameter/group" ]; then + ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" +fi + +if [ "$ownergroup" ]; then + echo chown -R "$ownergroup" "$destination" +fi diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 88019c92..94b06d63 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,73 +3,112 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using ``rsync`` +cdist-type__rsync - Mirror directories using rsync DESCRIPTION ----------- -The purpose of this type is to bring power of ``rsync`` into ``cdist``. +WARNING: This type is of BETA quality: + +- it has not been tested widely +- interfaces *may* change +- if there is a better approach to solve the problem -> the type may even vanish + +If you are fine with these constraints, please read on. + + +This cdist type allows you to mirror local directories to the +target host using rsync. Rsync will be installed in the manifest of the type. +If group or owner are giveng, a recursive chown will be executed on the +target host. + +A slash will be appended to the source directory so that only the contents +of the directory are taken and not the directory name itself. REQUIRED PARAMETERS ------------------- source - Source directory in local machine. - If source is directory, slash (``/``) will be added to source and destination paths. + Where to take files from OPTIONAL PARAMETERS ------------------- -destination - Destination directory. Defaults to ``$__object_id``. +group + Group to chgrp to. owner - Will be passed to ``rsync`` as ``--chown=OWNER``. - Read ``rsync(1)`` for more details. + User to chown to. -group - Will be passed to ``rsync`` as ``--chown=:GROUP``. - Read ``rsync(1)`` for more details. - -mode - Will be passed to ``rsync`` as ``--chmod=MODE``. - Read ``rsync(1)`` for more details. - -options - Defaults to ``--recursive --links --perms --times``. - Due to `bug in Python's argparse`_, value must be prefixed with ``\``. +destination + Use this as the base destination instead of the object id remote-user - Defaults to ``root``. + Use this user instead of the default "root" for rsync operations. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -option - Pass additional options to ``rsync``. - See ``rsync(1)`` for all possible options. - Due to `bug in Python's argparse`_, value must be prefixed with ``\``. +rsync-opts + Use this option to give rsync options with. + See rsync(1) for available options. + Only "--" options are supported. + Write the options without the beginning "--" + Can be specified multiple times. + + +MESSAGES +-------- +NONE EXAMPLES -------- + .. code-block:: sh - __rsync /var/www/example.com \ - --owner root \ - --group www-data \ - --mode 'D750,F640' \ - --source "$__files/example.com/www" + # You can use any source directory + __rsync /tmp/testdir \ + --source /etc + + # Use source from type + __rsync /etc \ + --source "$__type/files/package" + + # Allow multiple __rsync objects to write to the same dir + __rsync mystuff \ + --destination /usr/local/bin \ + --source "$__type/files/package" + + __rsync otherstuff \ + --destination /usr/local/bin \ + --source "$__type/files/package2" + + # Use rsync option --exclude + __rsync /tmp/testdir \ + --source /etc \ + --rsync-opts exclude=sshd_conf + + # Use rsync with multiple options --exclude --dry-run + __rsync /tmp/testing \ + --source /home/tester \ + --rsync-opts exclude=id_rsa \ + --rsync-opts dry-run + + +SEE ALSO +-------- +:strong:`rsync`\ (1) AUTHORS ------- -Ander Punnar +Nico Schottelius COPYING ------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. +Copyright \(C) 2015 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 64fa804e..9bd44c6d 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,3 +1,21 @@ #!/bin/sh -e +# +# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options deleted file mode 100644 index d967b110..00000000 --- a/cdist/conf/type/__rsync/parameter/default/options +++ /dev/null @@ -1 +0,0 @@ ---recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index 833e9bbe..ac2b2390 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,6 +1,4 @@ destination -group -mode -options owner +group remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index 01925a15..fdb7cd88 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -option +rsync-opts diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file deleted file mode 100755 index ec3d0fe8..00000000 --- a/cdist/conf/type/__sed/explorer/file +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -if [ ! -e "$file" ] -then - echo "$file does not exist" >&2 - exit 1 -fi - -cat "$file" diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote deleted file mode 100755 index f99c5a88..00000000 --- a/cdist/conf/type/__sed/gencode-remote +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -script="$( cat "$__object/parameter/script" )" - -if [ "$script" = '-' ] -then - script="$( cat "$__object/stdin" )" -fi - -# since stdin is not available in explorer, we pull file from target with explorer - -file_from_target="$__object/explorer/file" - -sed_cmd='sed' - -if [ -f "$__object/parameter/regexp-extended" ] -then - sed_cmd="$sed_cmd -E" -fi - -# do sed dry run, diff result and if no change, then there's nothing to do -# also redirect diff's output to stderr for debugging purposes - -if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 -then - exit 0 -fi - -# we can't use -i, because it's not posix, so we fly with tempfile and cp -# and we use cp because we want to preserve destination file's attributes - -# shellcheck disable=SC2016 -echo 'tmp="$__object/tempfile"' - -echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - -echo "$script" - -echo 'EOF' - -echo "cp \"\$tmp\" '$file'" - -# shellcheck disable=SC2016 -echo 'rm -f "$tmp"' - -echo 'change' >> "$__messages_out" - -if [ -f "$__object/parameter/onchange" ] -then - cat "$__object/parameter/onchange" -fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst deleted file mode 100644 index 86789363..00000000 --- a/cdist/conf/type/__sed/man.rst +++ /dev/null @@ -1,57 +0,0 @@ -cdist-type__sed(7) -================== - -NAME ----- -cdist-type__sed - Transform text files with ``sed`` - - -DESCRIPTION ------------ -Transform text files with ``sed``. - - -REQUIRED MULTIPLE PARAMETERS ----------------------------- -script - ``sed`` script. - If ``-`` then the script is read from ``stdin``. - - -OPTIONAL PARAMETERS -------------------- -file - Path to the file. Defaults to ``$__object_id``. - -onchange - Execute this command if ``sed`` changes file. - - -BOOLEAN PARAMETERS ------------------- -regexp-extended - Use extended regular expressions in the script. - Might not be supported with every ``sed`` version. - - -EXAMPLES --------- - -.. code-block:: sh - - __sed /tmp/foobar --script 's/foo/bar/' - - echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - - - -AUTHORS -------- -Ander Punnar - - -COPYING -------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean deleted file mode 100644 index 1ad75c5d..00000000 --- a/cdist/conf/type/__sed/parameter/boolean +++ /dev/null @@ -1 +0,0 @@ -regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional deleted file mode 100644 index fa86f917..00000000 --- a/cdist/conf/type/__sed/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -file -onchange diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple deleted file mode 100644 index 84f7e31d..00000000 --- a/cdist/conf/type/__sed/parameter/required_multiple +++ /dev/null @@ -1 +0,0 @@ -script diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group deleted file mode 100755 index a6cb3dfd..00000000 --- a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -e - -if grep -Eq '^ssl-cert:' /etc/group -then - echo 'present' -else - echo 'absent' -fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state deleted file mode 100755 index cc5aae0b..00000000 --- a/cdist/conf/type/__snakeoil_cert/explorer/state +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -e - -key_path="$( cat "$__object/parameter/key-path" )" - -if echo "$key_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - key_path="$( printf "$key_path" "$__object_id" )" -fi - -cert_path="$( cat "$__object/parameter/cert-path" )" - -if echo "$cert_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - cert_path="$( printf "$cert_path" "$__object_id" )" -fi - -if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] -then - echo 'absent' -else - echo 'present' -fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote deleted file mode 100755 index 8ffbfad1..00000000 --- a/cdist/conf/type/__snakeoil_cert/gencode-remote +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -e - -state="$( cat "$__object/explorer/state" )" - -if [ "$state" = 'present' ] -then - exit 0 -fi - -if [ -f "$__object/parameter/common-name" ] -then - common_name="$( cat "$__object/parameter/common-name" )" -else - common_name="$__object_id" -fi - -key_path="$( cat "$__object/parameter/key-path" )" - -if echo "$key_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - key_path="$( printf "$key_path" "$__object_id" )" -fi - -cert_path="$( cat "$__object/parameter/cert-path" )" - -if echo "$cert_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - cert_path="$( printf "$cert_path" "$__object_id" )" -fi - -key_type="$( cat "$__object/parameter/key-type" )" - -key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" - -case "$key_type" in - rsa:*) - echo "openssl genrsa -out '$key_path' $key_type_arg" - ;; - ec:*) - echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" - ;; -esac - -# shellcheck disable=SC2016 -echo 'csr_path="$( mktemp )"' - -echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" - -echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" - -# shellcheck disable=SC2016 -echo 'rm -f "$csr_path"' - -if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] -then - key_group='ssl-cert' -else - key_group='root' -fi - -echo "chmod 640 '$key_path'" - -echo "chown root '$key_path'" - -echo "chgrp $key_group '$key_path'" - -echo "chmod 644 '$cert_path'" - -echo "chown root '$cert_path'" - -echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst deleted file mode 100644 index b0b0a2e9..00000000 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ /dev/null @@ -1,61 +0,0 @@ -cdist-type__snakeoil_cert(7) -============================ - -NAME ----- -cdist-type__snakeoil_cert - Generate self-signed certificate - - -DESCRIPTION ------------ -The purpose of this type is to generate **self-signed** certificate and private key -for **testing purposes**. Certificate will expire in 3650 days. - -Certificate's and key's access bits will be ``644`` and ``640`` respectively. -If target system has ``ssl-cert`` group, then it will be used as key's group. -Use ``require='__snakeoil_cert/...' __file ...`` to override. - - -OPTIONAL PARAMETERS -------------------- -common-name - Defaults to ``$__object_id``. - -key-path - ``%s`` in path will be replaced with ``$__object_id``. - Defaults to ``/etc/ssl/private/%s.pem``. - -key-type - Possible values are ``rsa:$bits`` and ``ec:$name``. - For possible EC names see ``openssl ecparam -list_curves``. - Defaults to ``rsa:2048``. - -cert-path - ``%s`` in path will be replaced with ``$__object_id``. - Defaults to ``/etc/ssl/certs/%s.pem``. - - -EXAMPLES --------- -.. code-block:: sh - - __snakeoil_cert localhost-rsa \ - --common-name localhost \ - --key-type rsa:4096 - - __snakeoil_cert localhost-ec \ - --common-name localhost \ - --key-type ec:prime256v1 - - -AUTHORS -------- -Ander Punnar - - -COPYING -------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path deleted file mode 100644 index 4bbae089..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path +++ /dev/null @@ -1 +0,0 @@ -/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path deleted file mode 100644 index 86eb9359..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path +++ /dev/null @@ -1 +0,0 @@ -/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type deleted file mode 100644 index f13f8ada..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type +++ /dev/null @@ -1 +0,0 @@ -rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional deleted file mode 100644 index 76d08c0a..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/optional +++ /dev/null @@ -1,4 +0,0 @@ -common-name -key-path -key-type -cert-path diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index cbffde94..61c77fb9 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,7 +40,6 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" -rm -f "\$tmpfile" DONE } diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index 9694a64b..cec25746 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,7 +1,6 @@ #!/bin/sh -e # shellcheck disable=SC1090 -# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index bb1619a9..34aaca56 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ -| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +update-alternatives --display "$__object_id" 2>/dev/null \ + | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index d1087c75..6519e7c2 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/${__object_id:?}" ] + if [ ! -f "$altdir/$__object_id" ] then continue fi - link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" + link="$( awk 'NR==2' "$altdir/$__object_id" )" if [ -n "$link" ] then @@ -31,12 +31,9 @@ do fi done -if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] +if [ -z "$link" ] then - # NOTE: ignore error for dry-runs because a package providing the link - # might be managed by another cdist object (which wasn't executed, - # because dry run…). - echo "unable to get link for ${__object_id:?}" >&2 + echo "unable to get link for $__object_id" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 5cf4fa4b..fc304d5d 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,15 +1,11 @@ #!/bin/sh -e -path_is=$( - LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ - | awk '/link currently points to/ { print $5 }') +path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ + | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] +if [ -z "$path_is" ] then - # NOTE: ignore error for dry-runs because a package providing the - # alternative might be managed by another cdist object (which - # wasn't executed, because dry run…). - echo "unable to get current path for ${__object_id:?}" >&2 + echo "unable to get current path for $__object_id" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index b74a7ee8..59e015c5 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "${__object:?}/parameter/path" )" ] +if [ -f "$( cat "$__object/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e91ea78f..e393cdef 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,39 +18,37 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "${__object:?}/explorer/path_is" )" +path_is="$( cat "$__object/explorer/path_is" )" -path_should="$( cat "${__object:?}/parameter/path" )" +path_should="$( cat "$__object/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ - && [ -z "${__cdist_dry_run+dry run}" ] +if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name=${__object_id:?} +name="$__object_id" -if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" +alternatives="$( cat "$__object/explorer/alternatives" )" + +if ! echo "$alternatives" | grep -Fxq "$path_should" then - if [ -f "${__object:?}/parameter/install" ] + if [ ! -f "$__object/parameter/install" ] then - link="$( cat "${__object:?}/explorer/link" )" - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" - elif [ -z "${__cdist_dry_run+dry run}" ] - then - # NOTE: ignore error for dry-runs because a package providing the link - # to be installed might be managed by another cdist object (which - # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi + + link="$( cat "$__object/explorer/link" )" + + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" diff --git a/cdist/config.py b/cdist/config.py index 638fdf0e..56967ef6 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -537,7 +537,7 @@ class Config: objects_changed = False for cdist_object in self.object_list(): - if cdist_object.has_requirements_unfinished( + if cdist_object.requirements_unfinished( cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -548,7 +548,7 @@ class Config: self.object_prepare(cdist_object) objects_changed = True - if cdist_object.has_requirements_unfinished( + if cdist_object.requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -567,8 +567,7 @@ class Config: cargo = [] for cdist_object in self.object_list(): - if cdist_object.has_requirements_unfinished( - cdist_object.requirements): + if cdist_object.requirements_unfinished(cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -622,13 +621,12 @@ class Config: del cargo[:] for cdist_object in self.object_list(): - if cdist_object.has_requirements_unfinished( - cdist_object.requirements): + if cdist_object.requirements_unfinished(cdist_object.requirements): """We cannot do anything for this poor object""" continue if cdist_object.state == core.CdistObject.STATE_PREPARED: - if cdist_object.has_requirements_unfinished( + if cdist_object.requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -699,22 +697,20 @@ class Config: check for cycles. ''' graph = {} - - def _add_requirements(cdist_object, requirements): + for cdist_object in self.object_list(): obj_name = cdist_object.name if obj_name not in graph: graph[obj_name] = [] - - for requirement in cdist_object.requirements_unfinished( - requirements): - graph[obj_name].append(requirement.name) - - for cdist_object in self.object_list(): if cdist_object.state == cdist_object.STATE_DONE: continue - _add_requirements(cdist_object, cdist_object.requirements) - _add_requirements(cdist_object, cdist_object.autorequire) + for requirement in cdist_object.requirements_unfinished( + cdist_object.requirements): + graph[obj_name].append(requirement.name) + + for requirement in cdist_object.requirements_unfinished( + cdist_object.autorequire): + graph[obj_name].append(requirement.name) return graph_check_cycle(graph) def iterate_until_finished(self): @@ -792,19 +788,31 @@ class Config: def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" - self._handle_deprecation(cdist_object) - self.log.verbose("Preparing object %s", cdist_object.name) - self.log.verbose("Running manifest and explorers for %s", - cdist_object.name) - self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) - try: - self.manifest.run_type_manifest(cdist_object) - self.log.trace("[ORDER_DEP] Removing order dep files for %s", - cdist_object) - cdist_object.cleanup() - cdist_object.state = core.CdistObject.STATE_PREPARED - except cdist.Error as e: - raise cdist.CdistObjectError(cdist_object, e) + + if cdist_object.processing_preconditions_satisfied(): + self._handle_deprecation(cdist_object) + self.log.verbose("Preparing object %s", cdist_object.name) + self.log.verbose("Running manifest and explorers for %s", + cdist_object.name) + self.explorer.run_type_explorers(cdist_object, + transfer_type_explorers) + try: + self.manifest.run_type_manifest(cdist_object) + self.log.trace("[ORDER_DEP] Removing order dep files for %s", + cdist_object) + cdist_object.cleanup() + cdist_object.state = core.CdistObject.STATE_PREPARED + except cdist.Error as e: + raise cdist.CdistObjectError(cdist_object, e) + else: + cdist_object.state = core.CdistObject.STATE_DONE + cdist_object.skipped = True + cdist_object.skipped_reason = ( + "Processing preconditions not satisfied for object {0}: {1}, " + "skipping {0} object processing".format( + cdist_object.name, cdist_object.processing_preconditions) + ) + self.log.verbose(cdist_object.skipped_reason) def object_run(self, cdist_object): """Run gencode and code for an object""" @@ -821,9 +829,12 @@ class Config: cdist_object) if cdist_object.code_local or cdist_object.code_remote: cdist_object.changed = True + cdist_object.code_generated = True + else: + cdist_object.changed = False # Execute - if cdist_object.code_local or cdist_object.code_remote: + if cdist_object.changed: self.log.info("Processing %s", cdist_object.name) if not self.dry_run: if cdist_object.code_local: diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index bb3a65bd..653608cc 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -229,6 +229,8 @@ class CdistObject: requirements = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'require')) + processing_preconditions = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'onchange')) autorequire = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'autorequire')) parameters = fsproperty.DirectoryDictProperty( @@ -254,6 +256,12 @@ class CdistObject: # types children = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'children')) + code_generated = fsproperty.FileBooleanProperty( + lambda obj: os.path.join(obj.absolute_path, "code_generated")) + skipped = fsproperty.FileBooleanProperty( + lambda obj: os.path.join(obj.absolute_path, "skipped")) + skipped_reason = fsproperty.FileStringProperty( + lambda obj: os.path.join(obj.absolute_path, "skipped_reason")) def cleanup(self): try: @@ -280,7 +288,7 @@ class CdistObject: '{}: {}').format(self, error)) def requirements_unfinished(self, requirements): - """Return unsatisfied requirements""" + """Return state whether requirements are satisfied""" object_list = [] @@ -292,13 +300,23 @@ class CdistObject: return object_list - def has_requirements_unfinished(self, requirements): - """Return whether requirements are satisfied""" + def _processing_preconditions_satisfied_dfs(self, current): + # DFS (depth first search) for type that generated code. + obj = self.object_from_name(current) + if obj.code_generated: + return True + # As soon as one child that generated code is found, result with True. + for child in obj.children: + rv = self._processing_preconditions_satisfied_dfs(child) + if rv: + return True + return False - for requirement in requirements: - cdist_object = self.object_from_name(requirement) + def processing_preconditions_satisfied(self): + """Return state whether processing preconditions are satisfied""" - if cdist_object.state != self.STATE_DONE: + for onchange in self.processing_preconditions: + if self._processing_preconditions_satisfied_dfs(onchange): return True - return False + return len(self.processing_preconditions) == 0 diff --git a/cdist/emulator.py b/cdist/emulator.py index c55b47d2..dd6f9dc6 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -104,7 +104,7 @@ class Emulator: with flock.Flock(self.flock_path): self.setup_object() self.save_stdin() - self.record_requirements() + self.record_requirements_and_processing_preconditions() self.record_auto_requirements() self.record_parent_child_relationships() self.log.trace("Finished %s %s", self.cdist_object.path, @@ -294,30 +294,43 @@ class Emulator: except EnvironmentError as e: raise cdist.Error('Failed to read from stdin: {}'.format(e)) - def record_requirement(self, requirement): - """record requirement and return recorded requirement""" + def _record_dep(self, object_name, attr_name, msg): + """ + Record dependency for attr_name and return recorded value. + """ # Raises an error, if object cannot be created try: - cdist_object = self.cdist_object.object_from_name(requirement) + cdist_object = self.cdist_object.object_from_name(object_name) except core.cdist_type.InvalidTypeError as e: - self.log.error("%s requires object %s, but type %s does not" - " exist. Defined at %s", self.cdist_object.name, - requirement, e.name, self.object_source) + self.log.error("%s %s object %s, but type %s does not exist." + " Defined at %s", msg, self.cdist_object.name, + object_name, e.name, self.object_source) raise except core.cdist_object.MissingObjectIdError: - self.log.error("%s requires object %s without object id." - " Defined at %s", self.cdist_object.name, - requirement, self.object_source) + self.log.error("%s %s object %s without object id. Defined at %s", + self.cdist_object.name, msg, object_name, + self.object_source) raise - self.log.debug("Recording requirement %s for %s", - requirement, self.cdist_object.name) + self.log.debug("Recording %s in %s for %s", + object_name, attr_name, self.cdist_object.name) # Save the sanitised version, not the user supplied one # (__file//bar => __file/bar) # This ensures pattern matching is done against sanitised list - self.cdist_object.requirements.append(cdist_object.name) + attr_list = getattr(self.cdist_object, attr_name) + if cdist_object.name not in attr_list: + attr_list.append(cdist_object.name) + + def _record_requirement(self, requirement): + """record requirement and return recorded requirement""" + self._record_dep(requirement, 'requirements', 'requires') + + def _record_processing_precondition(self, onchange): + """record processing precondition and return t""" + self._record_dep(onchange, 'processing_preconditions', + 'has processing precondition') def _order_dep_on(self): return os.path.exists(self.order_dep_state_path) @@ -351,8 +364,25 @@ class Emulator: except FileNotFoundError: return [] - def record_requirements(self): - """Record requirements.""" + def _inject_dependency(self, lastcreatedtype, dep_var): + if dep_var in self.env: + if lastcreatedtype not in self.env[dep_var]: + self.env[dep_var] += " " + lastcreatedtype + else: + self.env[dep_var] = lastcreatedtype + + def _process_dep_var(self, func, dep_var): + if dep_var in self.env: + objects = self.env[dep_var] + self.log.debug("reqs for %s = %s", dep_var, objects) + for requirement in self._parse_require(objects): + # Ignore empty fields - probably the only field anyway + if len(requirement) == 0: + continue + func(requirement) + + def record_requirements_and_processing_preconditions(self): + """Record requirements and processing precondition.""" order_dep_on = self._order_dep_on() @@ -373,27 +403,23 @@ class Emulator: typeorder = self._read_typeorder_dep() # get the type created before this one lastcreatedtype = typeorder[-2].strip() - if 'require' in self.env: - if lastcreatedtype not in self.env['require']: - self.env['require'] += " " + lastcreatedtype - else: - self.env['require'] = lastcreatedtype - self.log.debug("Injecting require for" - " CDIST_ORDER_DEPENDENCY: %s for %s", - lastcreatedtype, self.cdist_object.name) + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, + self.cdist_object.name) + self._inject_dependency(lastcreatedtype, 'require') except IndexError: # if no second last line, we are on the first type, # so do not set a requirement pass - if "require" in self.env: - requirements = self.env['require'] - self.log.debug("reqs = %s", requirements) - for requirement in self._parse_require(requirements): - # Ignore empty fields - probably the only field anyway - if len(requirement) == 0: - continue - self.record_requirement(requirement) + self._process_dep_var(func=self._record_requirement, + dep_var='require') + # onchange implies require + self._process_dep_var(func=self._record_requirement, + dep_var='onchange') + self._process_dep_var(func=self._record_processing_precondition, + dep_var='onchange') def _parse_require(self, require): return re.split(r'[ \t\n]+', require) diff --git a/cdist/integration.py b/cdist/integration.py index 04470ea7..17b65f09 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.abc.Iterable): + elif isinstance(host, collections.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/log.py b/cdist/log.py index 62e457fe..113f3b4c 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,27 +36,25 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') - logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(self, msg, *args, **kwargs): - self.log(logging.VERBOSE, msg, args, **kwargs) +def _verbose(msg, *args, **kwargs): + logging.log(logging.VERBOSE, msg, *args, **kwargs) -logging.Logger.verbose = _verbose - +logging.verbose = _verbose logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(self, msg, *args, **kwargs): - self.log(logging.TRACE, msg, *args, **kwargs) +def _trace(msg, *args, **kwargs): + logging.log(logging.TRACE, msg, *args, **kwargs) -logging.Logger.trace = _trace +logging.trace = _trace class CdistFormatter(logging.Formatter): diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index ddbe4933..eca4cf13 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,98 +20,36 @@ # import logging -import sys -from datetime import datetime log = logging.getLogger("scan") -def run(scan, args): - # We run each component in a separate process since they - # must not block on each other. +# define this outside of the class to not handle scapy import errors by default +def commandline(args): + log.debug(args) + + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + print('cdist scan requires scapy to be installed') + processes = [] + if not args.mode: + # By default scan and trigger, but do not call any action + args.mode = ['scan', 'trigger', ] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interface, - sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces) t.start() processes.append(t) log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner( - autoconfigure='config' in args.mode, - interfaces=args.interface, - name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interfaces, args=args) s.start() processes.append(s) log.debug("Scanner started") for process in processes: process.join() - - -def list(scan, args): - s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) - hosts = s.list() - - # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + - # 6 colons. - ipv6_max_size = 8 * 4 + 10 - date_max_size = len(datetime.now().strftime(scan.datetime_format)) - name_max_size = 25 - - print("{} | {} | {} | {}".format( - 'name'.ljust(name_max_size), - 'address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size), - 'last configured'.ljust(date_max_size))) - print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) - for host in hosts: - last_seen = host.last_seen() - if last_seen: - last_seen = last_seen.strftime(scan.datetime_format) - else: - last_seen = '-' - - last_configured = host.last_configured() - if last_configured is not None: - last_configured = last_configured.strftime(scan.datetime_format) - else: - last_configured = '-' - - print("{} | {} | {} | {}".format( - host.name(default='-').ljust(name_max_size), - host.address().ljust(ipv6_max_size), - last_seen.ljust(date_max_size), - last_configured.ljust(date_max_size))) - - -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Set default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - - if 'config' in args.mode and args.name_mapper is None: - print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) - sys.exit(1) - - # Print known hosts and exit is --list is specified - do not start - # the scanner. - if args.list: - list(scan, args) - else: - run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 4a20f511..faee8a56 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,6 +19,38 @@ # # +# +# Interface to be implemented: +# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated +# scan: scan / listen for icmp6 replies +# trigger: send trigger to multicast +# config: configure newly detected hosts +# install: install newly detected hosts +# +# Scanner logic +# - save results to configdir: +# basedir = ~/.cdist/scan/ +# last_seen = ~/.cdist/scan//last_seen -- record unix time +# or similar +# last_configured = ~/.cdist/scan//last_configured -- record +# unix time or similar +# last_installed = ~/.cdist/scan//last_configured -- record +# unix time or similar +# +# +# +# +# cdist scan --list +# Show all known hosts including last seen flag +# +# Logic for reconfiguration: +# +# - record when configured last time +# - introduce a parameter --reconfigure-after that takes time argument +# - reconfigure if a) host alive and b) reconfigure-after time passed +# + + from multiprocessing import Process import os import logging @@ -29,84 +61,7 @@ import datetime import cdist.config -logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") -datetime_format = '%Y-%m-%d %H:%M:%S' - - -class Host(object): - def __init__(self, addr, outdir, name_mapper=None): - self.addr = addr - self.workdir = os.path.join(outdir, addr) - self.name_mapper = name_mapper - - os.makedirs(self.workdir, exist_ok=True) - - def __get(self, key, default=None): - fname = os.path.join(self.workdir, key) - value = default - if os.path.isfile(fname): - with open(fname, "r") as fd: - value = fd.readline() - return value - - def __set(self, key, value): - fname = os.path.join(self.workdir, key) - with open(fname, "w") as fd: - fd.write(f"{value}") - - def name(self, default=None): - if self.name_mapper is None: - return default - - fpath = os.path.join(os.getcwd(), self.name_mapper) - if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: - value = out.stdout.decode() - return (default if len(value) == 0 else value) - else: - return default - - def address(self): - return self.addr - - def last_seen(self, default=None): - raw = self.__get('last_seen') - if raw: - return datetime.datetime.strptime(raw, datetime_format) - else: - return default - - def last_configured(self, default=None): - raw = self.__get('last_configured') - if raw: - return datetime.datetime.strptime(raw, datetime_format) - else: - return default - - def seen(self): - now = datetime.datetime.now().strftime(datetime_format) - self.__set('last_seen', now) - - # XXX: There's no easy way to use the config module without feeding it with - # CLI args. Might as well call everything from scratch! - def configure(self): - target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target] - - fname = os.path.join(self.workdir, 'last_configuration_log') - with open(fname, "w") as fd: - log.debug("Executing: %s", cmd) - completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) - if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for \ - details.", cmd, completed_process.returncode, fname) - - now = datetime.datetime.now().strftime(datetime_format) - self.__set('last_configured', now) class Trigger(object): @@ -114,14 +69,12 @@ class Trigger(object): Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces, sleeptime, verbose=False): + def __init__(self, interfaces=None, verbose=False): self.interfaces = interfaces - - # Used by scapy / send in trigger/2. self.verbose = verbose - # Delay in seconds between sent ICMPv6EchoRequests. - self.sleeptime = sleeptime + # Wait 5 seconds before triggering again - FIXME: add parameter + self.sleeptime = 5 def start(self): self.processes = [] @@ -140,14 +93,9 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - try: - log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6( - dst="ff02::1%{}".format(interface) - ) / ICMPv6EchoRequest() - send(packet, verbose=self.verbose) - except Exception as e: - log.error("Could not send ICMPv6EchoRequest: %s", e) + packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() + log.debug("Sending request on %s", interface) + send(packet, verbose=self.verbose) class Scanner(object): @@ -155,62 +103,41 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, - name_mapper=None): + def __init__(self, interfaces=None, args=None, outdir=None): self.interfaces = interfaces - self.autoconfigure = autoconfigure - self.name_mapper = name_mapper - self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') - os.makedirs(self.outdir, exist_ok=True) - - self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) - if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), - host.address()) - else: - log.verbose("Host %s is alive", host.address()) + host = pkg['IPv6'].src + log.verbose("Host %s is alive", host) - host.seen() + dir = os.path.join(self.outdir, host) + fname = os.path.join(dir, "last_seen") - # Configure if needed. - if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + \ - self.config_delay < datetime.datetime.now(): - self.config(host) + now = datetime.datetime.now() - def list(self): - hosts = [] - for addr in os.listdir(self.outdir): - hosts.append(Host(addr, self.outdir, self.name_mapper)) + os.makedirs(dir, exist_ok=True) - return hosts + # FIXME: maybe adjust the format so we can easily parse again + with open(fname, "w") as fd: + fd.write(f"{now}\n") - def config(self, host): - if host.name() is None: - log.debug("config - could not resolve name for %s, aborting.", - host.address()) - return + def config(self): + """ + Configure a host - previous_config_process = self.running_configs.get(host.name()) - if previous_config_process is not None and \ - previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", - host.name()) + - Assume we are only called if necessary + - However we need to ensure to not run in parallel + - Maybe keep dict storing per host processes + - Save the result + - Save the output -> probably aligned to config mode - log.info("config - running against host %s (%s).", host.name(), - host.address()) - p = Process(target=host.configure()) - p.start() - self.running_configs[host.name()] = p + """ def start(self): self.process = Process(target=self.scan) @@ -221,9 +148,47 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - try: - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - except Exception as e: - log.error("Could not start listener: %s", e) + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + + +if __name__ == '__main__': + t = Trigger(interfaces=["wlan0"]) + t.start() + + # Scanner can listen on many interfaces at the same time + s = Scanner(interfaces=["wlan0"]) + s.scan() + + # Join back the trigger processes + t.join() + + # Test in my lan shows: + # [18:48] bridge:cdist% ls -1d fe80::* + # fe80::142d:f0a5:725b:1103 + # fe80::20d:b9ff:fe49:ac11 + # fe80::20d:b9ff:fe4c:547d + # fe80::219:d2ff:feb2:2e12 + # fe80::21b:fcff:feee:f446 + # fe80::21b:fcff:feee:f45c + # fe80::21b:fcff:feee:f4b1 + # fe80::21b:fcff:feee:f4ba + # fe80::21b:fcff:feee:f4bc + # fe80::21b:fcff:feee:f4c1 + # fe80::21d:72ff:fe86:46b + # fe80::42b0:34ff:fe6f:f6f0 + # fe80::42b0:34ff:fe6f:f863 + # fe80::42b0:34ff:fe6f:f9b2 + # fe80::4a5d:60ff:fea1:e55f + # fe80::77a3:5e3f:82cc:f2e5 + # fe80::9e93:4eff:fe6c:c1f4 + # fe80::ba69:f4ff:fec5:6041 + # fe80::ba69:f4ff:fec5:8db7 + # fe80::bad8:12ff:fe65:313d + # fe80::bad8:12ff:fe65:d9b1 + # fe80::ce2d:e0ff:fed4:2611 + # fe80::ce32:e5ff:fe79:7ea7 + # fe80::d66d:6dff:fe33:e00 + # fe80::e2ff:f7ff:fe00:20e6 + # fe80::f29f:c2ff:fe7c:275e diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 4b2bc3ba..29b68d90 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -207,6 +207,40 @@ class EmulatorTestCase(test.CdistTestCase): self.assertTrue(len(reqs) == 0) # if we get here all is fine + def test_onchange(self): + argv = ['__planet', 'erde'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + argv = ['__planet', 'mars'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + self.env['onchange'] = '__planet/erde __planet/mars' + argv = ['__file', '/tmp/cdisttest'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + # now load the objects and verify the require parameter of the objects + cdist_type = core.CdistType(self.local.type_path, '__planet') + erde_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'erde') + mars_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'mars') + cdist_type = core.CdistType(self.local.type_path, '__file') + file_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + '/tmp/cdisttest') + # now test the recorded exec prerequisites and requirements + self.assertTrue(len(erde_object.requirements) == 0) + self.assertTrue(len(mars_object.requirements) == 0) + self.assertTrue(len(file_object.requirements) > 0) + self.assertTrue(len(erde_object.processing_preconditions) == 0) + self.assertTrue(len(mars_object.processing_preconditions) == 0) + self.assertTrue(len(file_object.processing_preconditions) > 0) + self.assertEqual(list(file_object.requirements), + ['__planet/erde', '__planet/mars']) + self.assertEqual(list(file_object.processing_preconditions), + ['__planet/erde', '__planet/mars']) + # if we get here all is fine + class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 6bf935e8..09e9cc19 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.abc.MutableSequence): +class FileList(collections.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.abc.MutableSequence): self.__write(lines) -class DirectoryDict(collections.abc.MutableMapping): +class DirectoryDict(collections.MutableMapping): """A dict that stores it's items as files in a directory. """ diff --git a/docs/changelog b/docs/changelog index 00defc2a..03ae7566 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,61 +1,12 @@ Changelog --------- -7.0.0: 2022-07-31 - * Explorer machine_type: Rewrite (Dennis Camera) - * New type: __sed (Ander Punnar) - * New type: __haproxy_dualstack (Evilham and ungleich) - * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) - * Type __apt_source: Fix complaint about suite change (Matthias Stecher) - * Type __package_apt: Fix complaint about suite change (Matthias Stecher) - * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) - * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) - * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) - * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) - * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) - * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) - * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) - * Type __apt_source: Add signed-by parameter (Daniel Fancsali) - * Explorer: add support for checkpoint (Stephan Leemburg) - -6.9.8: 2021-08-24 - * Type __rsync: Rewrite (Ander Punnar) - * New type: __apt_pin (Daniel Fancsali) - * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) - * Core: Fix logging bug (Dennis Camera) - * Build: Improve Makefile compatibility (Evilham) - * Type __filesystem: Support ubuntu (Joachim Desroches) - * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) - * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) - * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) - * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) - * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) - -6.9.7: 2021-07-10 - * New type: __postgres_conf (Beni Ruef, Dennis Camera) - * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) - * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) - * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) - * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) - * Type __git: Fix group explorer (Ander Punnar) - * Type __pyvenv: Fix group explorer (Dennis Camera) - * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) - * Type __debconf_set_selections: Add state explorer (Dennis Camera) - * Core: Implement usable cdist scan (Timothée Floure) - * New type: __snakeoil_cert (Ander Punnar) - * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) - -6.9.6: 2021-04-20 +next: * Type __pyvenv: Fix user example in man page (Dennis Camera) * Core: config: Make local state directory available to custom remotes (Steven Armstrong * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) * Core: Maintain object relationship graph in cdist cache (Darko Poljak) - * Type __git: Fix numeric owner and group handling (Dennis Camera) - * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) - * Type __download: Make sum parameter optional (Ander Punnar) 6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) @@ -174,7 +125,7 @@ Changelog * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -239,7 +190,7 @@ Changelog * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1098,9 +1049,9 @@ Changelog * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org deleted file mode 100644 index 42b4f5c5..00000000 --- a/docs/dev/release-process.org +++ /dev/null @@ -1,90 +0,0 @@ -* Install requirements (Alpine) - - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ - py3-build twine -* Ensure your gpg setup works with the email used in the git commit! - - For me this is nico@nico-notebook.schottelius.org - - Signature / id is on nb2 -* Create ~/.pypirc -[distutils] - index-servers = - pypi - cdist - -[pypi] - username = __token__ - password = ... - -[cdist] - repository = https://upload.pypi.org/legacy/ - username = __token__ - password = ... - -* Add date in docs/changelog -* Run ./bin/cdist-build-helper -* TODO Move to "build" - - python3 -m build -* DONE git tag: when? -CLOSED: [2022-07-31 Sun 23:58] -** Asked during release process: ok -* DONE Pypi error with distutils: do not use distutils anymore -CLOSED: [2022-07-31 Sun 23:58] -python3 setup.py sdist upload -... -Creating tar archive -removing 'cdist-7.0.0' (and everything under it) -running upload -Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ -Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. -error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. -(venv2) [22:50] nb2:cdist% - -* DONE Pypi error with twine: fixed in twine 4.0.1 -CLOSED: [2022-07-31 Sun 23:58] - -Seeing: - -(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* -Uploading distributions to https://upload.pypi.org/legacy/ -Traceback (most recent call last): - File "/usr/bin/twine", line 8, in - sys.exit(main()) - File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main - result = cli.dispatch(sys.argv[1:]) - File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch - return main(args.args) - File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main - return upload(upload_settings, parsed_args.dists) - File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload - resp = repository.upload(package) - File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload - resp = self._upload(package) - File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload - data = package.metadata_dictionary() - File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary - "dynamic": meta.dynamic, -AttributeError: 'Wheel' object has no attribute 'dynamic' - - -Fix: - - -(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* -⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. -Uploading distributions to https://upload.pypi.org/legacy/ -Uploading cdist-7.0.0-py3-none-any.whl -100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s -Uploading cdist-7.0.0.tar.gz -100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s - -View at: -https://pypi.org/project/cdist/7.0.0/ -* TODO cdist web - - on staticweb-2022 - - Should be moved to sftp/k8s - - - Manual steps: - - ~/bin/permissions.public html/ - rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ - ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest" diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 390ab9ec..18863145 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx with the rtd theme (for building html docs and/or the man pages) + * sphinx (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst index 2e49a721..7c3120bb 100644 --- a/docs/src/cdist-manifest.rst +++ b/docs/src/cdist-manifest.rst @@ -154,6 +154,16 @@ You can find a more in depth description of the flow execution of manifests in `cdist execution stages `_ and of how types work in `cdist type `_. +Processing preconditions +------------------------ +If you want to describe that something should be processed only if specified +objects changed target's state, i.e. specified objects generated either +"code-local" or "code-remote", or both, just setup the variable "onchange" to +contain processing preconditions. "onchange" is specified in the same way as +"require", and it implies "require". If processing preconditions are not +satisfied then it means that object's explorers are not run, manifest is not +run, and gencode scripts are not run. Object processing is skipped. + Create dependencies from execution order ----------------------------------------- You can tell cdist to execute all types in the order in which they are created @@ -375,3 +385,13 @@ Dependencies defined by execution order work as following: require="__some_type_somewhere/id __sample_type/1" __sample_type 2 require="__sample_type/2" __example_type 23 __not_in_order_type 42 + +This example makes use of processing preconditions: + +.. code-block:: sh + + __foo bar + __spam eggs + # execute __baz type only if any of __foo/bar, __spam/eggs has changed + # target's state, i.e. code-local or code-remote is generated (or both) + onchange='__foo/bar __spam/eggs' __baz diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index c0ac2c3e..01f1b963 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -314,6 +314,9 @@ The following environment variables influence the behaviour of cdist: require Setup dependencies between objects (see \`cdist manifest \`_). +onchange + Setup processing preconditions (implies dependencies) between objects (see `cdist manifest `_). + __cdist_log_level cdist log level value. One of: diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst deleted file mode 100644 index 86b7fab6..00000000 --- a/docs/src/cdist-scan.rst +++ /dev/null @@ -1,84 +0,0 @@ -Scan -===== - -Description ------------ -Runs cdist as a daemon that discover/watch on hosts and reconfigure them -periodically. It is especially useful in netboot-based environment where hosts -boot unconfigured, and to ensure your infrastructure stays in sync with your -configuration. - -This feature is still consider to be in **beta** stage, and only operate on -IPv6 (including link-local). - -Usage (Examples) ----------------- - -Discover hosts on local network and configure those whose name is resolved by -the name mapper script. - -.. code-block:: sh - - $ cdist scan --beta --interface eth0 \ - --mode scan --name-mapper path/to/script \ - --mode trigger --mode config - -List known hosts and exit. - -.. code-block:: sh - - $ cdist scan --beta --list --name-mapper path/to/script - -Please refer to `cdist(1)` for a detailed list of parameters. - -Modes ------ - -The scanner has 3 modes that can be independently toggled. If the `--mode` -parameter is not specified, only `tigger` and `scan` are enabled (= hosts are -not configured). - -trigger - Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to - trigger detection by the `scan` module. - -scan - Watch for incoming ICMPv6 replies and optionally configure detected hosts. - -config - Enable configuration of hosts detected by `scan`. - -Name Mapper Script ------------------- - -The name mapper script takes an IPv6 address as first argument and writes the -resolved name to stdout - if any. The script must be executable. - -Simplest script: - -.. code-block:: sh - - #!/bin/sh - - case "$1" in - "fe80::20d:b9ff:fe57:3524") - printf "my-host-01" - ;; - "fe80::7603:bdff:fe05:89bb") - printf "my-host-02" - ;; - esac - -Resolving name from `PTR` DNS record: - -.. code-block:: sh - - #!/bin/sh - - for cmd in dig sed; do - if ! command -v $cmd > /dev/null; then - exit 1 - fi - done - - dig +short -x "$1" | sed -e 's/.$//' diff --git a/docs/src/conf.py b/docs/src/conf.py index a3dfafca..47765413 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2021' +copyright = 'ungleich GmbH 2020' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/src/index.rst b/docs/src/index.rst index 369d5309..31c044dc 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,7 +34,6 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos - cdist-scan cdist-integration cdist-reference cdist-best-practice diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 599ec3b7..0ecb4a61 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,9 +88,6 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] - cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] - [-d CONFIG_DELAY] [-t TRIGGER_DELAY] - DESCRIPTION ----------- @@ -644,31 +641,6 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. -SCAN ----- - -Runs cdist as a daemon that discover/watch on hosts and reconfigure them -periodically. - -**-I INTERFACE, --interfaces INTERFACE** - Interface to listen on. Can be specified multiple times. - -**-m MODE, --mode MODE** - Scanner components to enable. Can be specified multiple time to enable more - than one component. Supported modes are: scan, trigger and config. Defaults - to tiggger and scan. - -**--name-mapper PATH_TO_SCRIPT** - Path to script used to resolve a remote host name from an IPv6 address. - -**--list** - List known hosts and exit. - -**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** - How long (seconds) to wait before reconfiguring after last try (config mode only). - -**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** - How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION -------------