diff --git a/Makefile b/Makefile index 3712511c..89286310 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=make -C $(DOCS_SRC_DIR) man -SPHINXH=make -C $(DOCS_SRC_DIR) html -SPHINXC=make -C $(DOCS_SRC_DIR) clean +SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man +SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html +SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/README.md b/README.md index de6901c7..a468dd86 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ freenode +IRC: ``#cdist`` @ [libera](https://libera.chat) Matrix: ``#cdist:ungleich.ch`` -Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist +Matrix and IRC are bridged. diff --git a/bin/cdist b/bin/cdist index ddaffa7f..adb06a8d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -72,9 +72,11 @@ def commandline(): if __name__ == "__main__": - if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: - print('Python >= {} is required on the source host.'.format( - cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) + if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION: + print( + 'Python >= {} is required on the source host.'.format( + ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))), + file=sys.stderr) sys.exit(1) exit_code = 0 diff --git a/cdist/__init__.py b/cdist/__init__.py index 44366cd0..31d49889 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" -MIN_SUPPORTED_PYTHON_VERSION = '3.5' +MIN_SUPPORTED_PYTHON_VERSION = (3, 5) class Error(Exception): diff --git a/cdist/argparse.py b/cdist/argparse.py index 88759d7b..f17315e7 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,19 +485,31 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger']) + choices=['scan', 'trigger', 'config']) + parser['scan'].add_argument( + '--list', + action='store_true', + help='List the known hosts and exit') parser['scan'].add_argument( '--config', action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interfaces', - action='append', default=[], + '-I', '--interface', + action='append', default=[], required=True, help='On which interfaces to scan/trigger') parser['scan'].add_argument( - '-d', '--delay', - action='store', default=3600, - help='How long to wait before reconfiguring after last try') + '--name-mapper', + action='store', default=None, + help='Map addresses to names, required for config mode') + parser['scan'].add_argument( + '-d', '--config-delay', + action='store', default=3600, type=int, + help='How long (seconds) to wait before reconfiguring after last try') + parser['scan'].add_argument( + '-t', '--trigger-delay', + action='store', default=5, type=int, + help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: @@ -533,10 +545,10 @@ def parse_and_configure(argv, singleton=True): log = logging.getLogger("cdist") - log.verbose("version %s" % cdist.VERSION) - log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(cfg.get_config())) - log.trace('configured args: {}'.format(args)) + log.verbose("version %s", cdist.VERSION) + log.trace('command line args: %s', cfg.command_line_args) + log.trace('configuration: %s', cfg.get_config()) + log.trace('configured args: %s', args) check_beta(vars(args)) diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 1c84f4d7..00646c75 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,8 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Daniel Heule (hda at sfs.biz) -# 2014 Thomas Oettli (otho at sfs.biz) -# 2020 Evilham (contact at evilham.com) +# 2021 Dennis Camera (cdist at dtnr.ch) # # This file is part of cdist. # @@ -19,91 +17,1019 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# This explorer tries to determine what type of machine the target to be +# configured is (container, virtual machine, bare-metal). +# +# It will print one line for each layer it can detect. +# The format of all lines is: TYPE[ VERB VENDOR] +# +# VERB does not have a special meaning, it is just for better readability. +# +# e.g. +# container +# container on lxc +# virtual by kvm-spapr +# +# The third word of each line (except the first) can be composed of different +# parts concatenated with a `-' (minus) character, with each component being +# a specification of the previous, e.g.: +# - lxc-libvirt (LXC container, managed by libvirt) +# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) +# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) +# +# If this explorer cannot collect enough information about virtualization it +# will fall back to 'physical'. +# -os=$("$__explorer/os") +# Add /sbin and /usr/sbin to the path so we can find system +# binaries like dmidecode. +PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' +PATH="/sbin:/usr/sbin:${PATH}" +export PATH -vendor_string_to_machine_type() { - for vendor in vmware bochs kvm qemu virtualbox bhyve; do - if echo "${1}" | grep -q -i "${vendor}"; then - if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then - vendor="kvm" - fi - echo "virtual_by_${vendor}" - exit - fi - done +arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') +uname_s=$(uname -s) + + +is_command() { command -v "$1" >/dev/null 2>&1; } + +files_same() { + # shellcheck disable=SC2012 + LC_ALL=C df -P "$1" "$2" 2>/dev/null | { + read -r _ # skip header line + read -r fs1 _ _ _ _ mp1 + read -r fs2 _ _ _ _ mp2 + test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 + } && + ls -1Ldi "$1" "$2" 2>/dev/null | { + read -r ino1 _ + read -r ino2 _ + test "${ino1}" = "${ino2}" || return 1 + } } -case "$os" in - "freebsd") - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" - if [ -n "${vm_guest}" ]; then - if [ "${vm_guest}" = "none" ]; then - echo "physical" - exit - fi - echo "virtual_by_${vm_guest}" - exit - fi - ;; +is_oneof() ( + x=$1; shift + for y + do + test "${x}" = "${y}" || continue + return 0 + done + return 1 +) - "openbsd") - # OpenBSD can also use the sysctl's: hw.vendor or hw.product. - # Note we can be reasonably sure about a machine being virtualised - # as long as we can identify the virtualisation technology. - # But not so much about it being physical... - # Patches are welcome / reach out if you have better ideas. - for sysctl in hw.vendor hw.product; do - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" - done - ;; +tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } - *) - # Defaulting to linux for compatibility with previous cdist behaviour +# shellcheck disable=SC2086 +glob_exists() { set -- $1; test -e "$1"; } - if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then - echo openvz - exit - fi +get_dmi_field() { + if is_oneof "${uname_s}" NetBSD + then + case $1 + in + (system-manufacturer) _mib=machdep.dmi.system-vendor ;; + (system-product-name) _mib=machdep.dmi.system-product ;; + (system-version|system-uuid) _mib=machdep.dmi.$1 ;; + (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; + (biod-release-date) _mib=machdep.dmi.bios-date ;; + (*) _mib= ;; + esac - if [ -e "/proc/1/environ" ] && - tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then - echo lxc - exit - fi + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi - if [ -r /proc/cpuinfo ]; then - # this should only exist on virtual guest machines, - # tested on vmware, xen, kvm, bhyve - if grep -q "hypervisor" /proc/cpuinfo; then - # this file is aviable in xen guest systems - if [ -r /sys/hypervisor/type ]; then - if grep -q -i "xen" /sys/hypervisor/type; then - echo virtual_by_xen - exit - fi - else - for vendor_file in /sys/class/dmi/id/product_name \ - /sys/class/dmi/id/sys_vendor \ - /sys/class/dmi/id/chasis_vendor; do - if [ -r ${vendor_file} ]; then - # This exits if we can make a reasonable judgement - vendor_string_to_machine_type "$(cat "${vendor_file}")" - fi - done - fi - echo "virtual_by_unknown" - exit - else - echo "physical" - exit - fi - fi - ;; -esac + if is_command dmidecode + then + dmidecode -s "$1" + elif test -d "${dmi_sysfs-}" + then + case $1 + in + (system-manufacturer) _filename=sys_vendor ;; + (system-product-name) _filename=product_name ;; + (*) _filename=$(echo "$1" | tr - _) ;; + esac + if test -r "${dmi_sysfs-}/${_filename}" + then + cat "${dmi_sysfs}/${_filename}" + fi + unset _filename + elif test "${uname_s}" = OpenBSD + then + # NOTE: something similar to system-manufacutrer and system-product-name + # is available on OpenBSD in sysctl + case $1 + in + (system-manufacturer) _mib=hw.vendor ;; + (system-product-name) _mib=hw.product ;; + (*) _mib= ;; + esac -echo "unknown" + test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return + fi + + return 1 +} + +has_cpuinfo() { test -e /proc/cpuinfo; } + +get_sysctl() { + is_command sysctl && sysctl -n "$1" 2>/dev/null +} + +detected_layer() { + test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" +} + + +# Check for chroot + +has_chroot_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' +} + +check_chroot_systemd() { + systemd-detect-virt -r +} + +has_chroot_debian_ischroot() { + is_command ischroot +} + +check_chroot_debian_ischroot() { + ischroot --default-false +} + +has_chroot_procfs() { + test -d /proc/ +} + +check_chroot_procfs() ( + is_chroot=false # default + if test -e /proc/1/root && ! files_same /proc/1/root / + then + is_chroot=true + fi + if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo + then + has_mountinfo=true + cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true + fi + + if ${is_chroot} + then + # try to determine where the chroot has been mounted + rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') + + if test -e "${rootdev}" + then + # escape chroot to determine where the device containing the + # chroot's / is mounted + rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') + + # shellcheck disable=SC2012 + root_ino=$(ls -1di / | awk '{print $1}') + + # escape chroot and find mount point by inode + chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" + elif ${has_mountinfo} + then + while read -r mntid _ _ _ cmntpnt _ + do + read -r _ _ _ _ hmntpnt _ <<-EOF + $(grep -e "^$((mntid)) " /proc/1/mountinfo) + EOF + printf '%s\n' "${hmntpnt%${cmntpnt}}" + done /dev/null) && + case ${_ctengine} + in + (''|'none') + return 1 ;; + ('container-other') + return 0 ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + echo "${_ctengine}" ;; + esac +) + +has_ct_pid_1() { + test -r /run/systemd/container -o -r /proc/1/environ +} + +translate_container_name() { + case $1 + in + ('lxc') + echo lxc ;; + ('lxc-libvirt') + echo lxc-libvirt ;; + ('podman') + echo podman ;; + ('systemd-nspawn') + echo systemd_nspawn ;; + (*) + return 1 ;; + esac + return 0 +} + +check_ct_pid_1() { + if test -r /run/systemd/container + then + translate_container_name "$(head -n1 /run/systemd/container)" \ + && return 0 + fi + + if test -r /proc/1/environ + then + translate_container_name "$( + LC_ALL=C tr '\000' '\n' /dev/null + then + # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 + echo wsl + elif test -d /var/.cagefs + then + # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs + # CageFS is not "really" a container, but it isn't a chroot either. + echo cagefs + elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status + then + # Linux-VServer + if grep -q -x -F 'VxID: 0' /proc/self/status + then + # host + return 1 + else + # guest + echo linux_vserver + fi + else + return 1 + fi +} + +check_ct_os_specific() ( + if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 + then + # FreeBSD jail + echo jail + return 0 + fi + + if is_command zonename && test "$(zonename)" != global + then + # Solaris zone + echo zone + return 0 + fi + + return 1 +) + + +# Check for hypervisor + +guess_hypervisor_from_cpu_model() { + case $1 + in + (*\ KVM\ *) + echo kvm ;; + (*\ QEMU\ *|QEMU\ *) + echo qemu ;; + (*) + return 1 ;; + esac +} + +has_vm_systemd() { + is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' +} + +check_vm_systemd() ( + _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && + case ${_hypervisor} + in + (''|'none') + return 1 ;; + ('amazon') + echo aws ;; + ('bochs') + echo kvm ;; + ('microsoft') + # assumption + echo hyperv ;; + ('oracle') + echo virtualbox ;; + ('vm-other') + return 0 ;; + (*) + echo "${_hypervisor}" ;; + esac +) + +has_vm_cpuinfo() { has_cpuinfo; } + +check_vm_cpuinfo() { + if grep -q -F 'User Mode Linux' /proc/cpuinfo \ + || grep -q -F 'UML' /proc/cpuinfo + then + # User Mode Linux + echo uml + elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo + then + # IBM PowerVM Lx86 (Linux/x86 emulator) + echo powervm_lx86 + elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo + then + # IBM SystemZ (S/390) + if test -f /proc/sysinfo + then + if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo + then + echo kvm-s390 + return 0 + elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo + then + echo zvm + return 0 + elif grep -q -e '^LPAR ' /proc/sysinfo + then + echo zvm-lpar + return 0 + fi + fi + return 1 + else + if grep -q -e '^model name.*:' /proc/cpuinfo + then + sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ + | while read -r _cpu_model + do + guess_hypervisor_from_cpu_model "${_cpu_model}" + done \ + | sort \ + | uniq -c \ + | awk ' + { if ($1 > most_c) { most_c = $1; most_s = $2 } } + END { + if (most_s) print most_s + exit !most_s + }' \ + && return 0 + fi + return 1 + fi +} + +check_vm_arch_specific() { + case ${arch} + in + (ppc64|ppc64le) + # Check PPC64 LPAR, KVM + + # example /proc/cpuinfo line indicating 'not baremetal' + # platform : pSeries + # + # example /proc/ppc64/lparcfg systemtype line + # system_type=IBM pSeries (emulated by qemu) + + if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo + then + if test -e /proc/ppc64/lparcfg + then + # Assume LPAR, now detect shared or dedicated + if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg + then + echo powervm-shared + return 0 + else + echo powervm-dedicated + return 0 + fi + fi + fi + ;; + (sparc*) + # Check for SPARC LDoms + + if test -e /dev/mdesc + then + if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp + then + # control LDom + return 1 + else + # guest LDom + echo ldom-sparc + fi + + # MDPROP=/usr/lib/ldoms/mdprop.py + # if test -x "${MDPROP}" + # then + # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" + # then + # echo ldoms-root + # echo ldoms-io + # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" + # then + # echo ldoms-io + # fi + # fi + return 0 + fi + ;; + (i?86|x86*|amd64|i86pc) + # VMM CPUID flag denotes that this system is running under a VMM + if is_oneof "${uname_s}" Darwin + then + get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ + && return 0 + fi + if has_cpuinfo \ + && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo + then + return 0 + fi + ;; + (ia64) + if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend + then + # PV-on-HVM drivers installed in a Xen guest + echo xen-hvm + return 0 + fi + ;; + esac + return 1 +} + +has_vm_dmi() { + # Check for various products in SMBIOS/DMI. + # Note that DMI doesn't exist on all architectures (only x86 and some ARM). + # On other architectures the $dmi variable will be empty. + + if test -d /sys/class/dmi/id/ + then + dmi_sysfs=/sys/class/dmi/id + elif test -d /sys/devices/virtual/dmi/id/ + then + dmi_sysfs=/sys/devices/virtual/dmi/id + fi + + # shellcheck disable=SC2015 + { + is_command dmidecode \ + && ( + # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line + dmi_out=$(dmidecode 2>&1) \ + && ! printf '%s\n' "${dmi_out}" \ + | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' + ) \ + || test -d "${dmi_sysfs}" + } +} + +check_vm_dmi() { + case $(get_dmi_field system-product-name) + in + (*.metal) + if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' + then + # AWS EC2 bare metal -> no virtualisation + return 1 + fi + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('Google Compute Engine') + echo gce + return 0 + ;; + ('RHEV Hypervisor') + # Red Hat Enterprise Virtualization + echo rhev + return 0 + ;; + ('KVM'|'Bochs'|'KVM Virtual Machine') + echo kvm + return 0 + ;; + ('Parallels Virtual Platform') + echo parallels + return 0 + ;; + ('VirtualBox') + echo virtualbox + return 0 + ;; + ('VMware Virtual Platform') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field system-manufacturer) + in + ('Alibaba'*) + case $(get_dmi_field system-product-name) + in + ('Alibaba Cloud ECS') + echo alibaba-ecs + ;; + (*) + echo alibaba + ;; + esac + return 0 + ;; + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('innotek GmbH'|'Oracle Corporation') + echo virtualbox + return 0 + ;; + ('Joyent') + if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' + then + # SmartOS KVM + echo kvm-smartdc_hvm + return 0 + fi + ;; + ('Microsoft Corporation'*) + if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' + then + if test -e /proc/irq/7/hyperv \ + || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null + then + echo hyperv + return 0 + fi + + case $(get_dmi_field system-version) + in + (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) + echo virtualpc + return 0 + ;; + (*) + echo hyperv + return 0 + ;; + esac + fi + ;; + ('Nutanix') + # Nutanix AHV. Similar to KVM. + if test "$(get_dmi_field system-product-name)" = 'AHV' + then + echo nutanix_ahv + return 0 + fi + ;; + ('oVirt') + echo ovirt + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('QEMU') + echo qemu + return 0 + ;; + ('VMware, Inc.') + echo vmware + return 0 + ;; + esac + + case $(get_dmi_field bios-vendor) + in + ('Amazon EC2') + # AWS on bare-metal or KVM + echo aws-ec2 + return 0 + ;; + ('BHYVE') + echo bhyve + return 0 + ;; + ('innotek GmbH') + echo virtualbox + return 0 + ;; + ('Parallels Software International Inc.') + echo parallels + return 0 + ;; + ('Xen') + if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' + then + # AWS on Xen + echo aws-xen + return 0 + fi + ;; + esac + + return 1 +} + +check_vm_hyp_specific() { + if is_command vmware-checkvm && vmware-checkvm >/dev/null + then + # vmware-checkvm is provided by VMware's open-vm-tools + echo vmware + return 0 + elif test -d /proc/xen + then + test -r /proc/xen/capabilities && + if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null + then + # Xen dom0 + return 1 + else + # Xen domU + echo xen + return 0 + fi + fi + return 1 +} + +has_vm_dt() { + # OpenFirmware/Das U-Boot device-tree + test -d /proc/device-tree +} + +check_vm_dt() { + case ${arch} + in + (arm|aarch64) + if test -r /proc/device-tree/hypervisor/compatible + then + if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible + then + echo xen + return 0 + elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible + then + # e.g. VMware ESXi on ARM + echo vmware + return 0 + fi + fi + if glob_exists /proc/device-tree/fw-cfg@*/compatible + then + # qemu,fw-cfg-mmio + sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 + return 0 + fi + if grep -q -F 'dummy-virt' /proc/device-tree/compatible + then + echo lkvm + return 0 + fi + ;; + (ppc64*) + if test -d /proc/device-tree/hypervisor \ + && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible + then + # We are running as a spapr KVM guest on ppc64 + echo kvm-spapr + return 0 + fi + if test -r /proc/device-tree/ibm,partition-name \ + && test -r /proc/device-tree/hmc-managed\? \ + && test -r /proc/device-tree/chosen/qemu,graphic-width + then + echo powervm + fi + ;; + esac + return 1 +} + +has_vm_sys_hypervisor() { + test -d /sys/hypervisor/ +} + +check_vm_sys_hypervisor() { + test -r /sys/hypervisor/type && + case $(head -n1 /sys/hypervisor/type) + in + (xen) + # Ordinary kernel with pv_ops. There does not seem to be + # enough information at present to tell whether this is dom0 + # or domU. + echo xen + return 0 + ;; + esac + return 1 +} + +check_vm_os_specific() { + _hyp_generic=false + + case ${uname_s} + in + (Darwin) + if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ + && test "${hv_vmm_present}" -ne 0 + then + _hyp_generic=true + fi + ;; + (FreeBSD) + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest=$(get_sysctl kern.vm_guest | tolower) && + case ${vm_guest} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${vm_guest} + in + (hv) echo hyperv ;; + (vbox) echo virtualbox ;; + (*) echo "${vm_guest}" ;; + esac + return 0 + ;; + esac + ;; + (NetBSD) + machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && + case ${machdep_hv} + in + (none) ;; + (generic) _hyp_generic=true ;; + (*) + # kernel could detect hypervisor + case ${machdep_hv} + in + (hyper-v) echo hyperv ;; + (xenhvm*) echo xen-hvm ;; + (xenpv*) echo xen-pv ;; + (xen*) echo xen ;; + (*) echo "${machdep_hv}" ;; + esac + return 0 + ;; + esac + ;; + (OpenBSD) + if is_command hostctl && glob_exists /dev/pvbus[0-9]* + then + for _pvbus in /dev/pvbus[0-9]* + do + _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue + case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') + in + (KVM) echo kvm ;; + (Hyper-V) echo hyperv ;; + (VMware) echo vmware ;; + (Xen) echo xen ;; + (bhyve) echo bhyve ;; + (OpenBSD) echo openbsd_vmm ;; + esac + return 0 + done + fi + ;; + (SunOS) + diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') + # NOTE: Don't use -e or -F in Solaris grep + if printf '%s\n' "${diag_conf}" | grep -q -i QEMU + then + echo qemu + return 0 + elif printf '%s\n' "${diag_conf}" | grep -q -i VMware + then + echo vmware + return 0 + fi + ;; + (Linux) + if is_command dmesg + then + while read -r line + do + case ${line} + in + ('Booting paravirtualized kernel on ') + case $(expr "${line}" : '.* kernel on \(.*\)') + in + ('Xen') + echo xen-pv; return 0 ;; + ('bare hardware') + return 1 ;; + esac + ;; + ('Hypervisor detected') + case $(expr "${line}" : '.*: *\(.*\)') + in + ('ACRN') + echo acrn ;; + ('Jailhouse') + echo jailhouse ;; + ('KVM') + echo kvm ;; + ('Microsoft Hyper-V') + echo hyperv ;; + ('VMware') + echo vmware ;; + ('Xen HVM') + echo xen-hvm ;; + ('Xen PV') + echo xen-pv ;; + esac + return 0 + ;; + (lpar:*' under hypervisor') + return 0 ;; + esac + done <<-EOF + $(dmesg 2>/dev/null | awk ' + /Booting paravirtualized kernel on / + /Hypervisor detected: / + /lpar: .* under hypervisor/ + ') + EOF + fi + esac + + # Try to guess hypervisor based on CPU model (sysctl hw.model if available) + if cpu_model=$(get_sysctl hw.model) + then + guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 + fi + + if ${_hyp_generic} + then + # cannot say which hypervisor, but one was detected + return 0 + else + return 1 + fi +} + +run_stage() { + if type "has_$1_$2" >/dev/null 2>&1 + then + "has_$1_$2" + else + true + fi \ + && "check_$1_$2" +} + + +# Execute chroot stages + +for stage in \ + procfs debian_ischroot systemd +do + chrootpnt=$(run_stage chroot ${stage}) || continue + is_chrooted=true + detected_layer 'chroot' + if test -n "${chrootpnt}" + then + echo chroot at "${chrootpnt}" + break + fi +done +if ${is_chrooted:-false} && test -z "${chrootpnt}" +then + # could determine chroot, but not its mount point + echo chroot +fi + + +# Execute container stages + +for stage in \ + systemd pid_1 cgroup files os_specific +do + ctengine=$(run_stage ct ${stage}) || continue + detected_layer 'container' + is_contained=true + if test -n "${ctengine}" + then + echo container on "${ctengine}" + break + fi +done +if ${is_contained:-false} && test -z "${ctengine}" +then + # none of the stages could determine the specific container engine, but + # we are running in some container. + echo container +fi + + +# Execute virtual machine / hypervisor stages + +for stage in \ + systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific +do + hypervisor=$(run_stage vm ${stage}) || continue + detected_layer 'virtual machine' + is_virtual=true + if test -n "${hypervisor}" + then + echo virtual by "${hypervisor}" + break + fi +done +if ${is_virtual:-false} && test -z "${hypervisor}" +then + # none of the stages could determine the specific hypervisor, but + # we are virtual. + echo virtual +fi + + +# Fallback + +detected_layer physical diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 63aba9c6..c6d113cf 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,19 +27,18 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { print $1 * 1000 } - $2 == "MB" { print $1 * 1000 * 1000 } - $2 == "GB" { print $1 * 1000 * 1000 * 1000 } - $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } - $2 == "kiB" { print $1 * 1024 } - $2 == "MiB" { print $1 * 1024 * 1024 } - $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } - $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' + $2 == "kB" { printf "%.f\n", ($1 * 1000) } + $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } + $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } + $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } + $2 == "kiB" { printf "%.f\n", ($1 * 1024) } + $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } + $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } + $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' } bytes2kib() { - set -- "$(cat)" - test "$1" -gt 0 && echo $(($1 / 1024)) + awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' } diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 3b02dedd..bbc9e4f0 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,12 +18,22 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# # All os variables are lower case # -# -case "$("$__explorer/os")" in +rc_getvar() { + awk -F= -v varname="$2" ' + function unquote(s) { + if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) + return substr(s, 2, length(s) - 2) + else + return s + } + $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" +} + +case $("${__explorer:?}/os") +in amazon) cat /etc/system-release ;; @@ -43,6 +54,8 @@ case "$("$__explorer/os")" in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in + trixie) echo 12.99 ;; + bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; @@ -50,7 +63,7 @@ case "$("$__explorer/os")" in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) exit 1 + *) echo 99.99 ;; esac ;; *) @@ -59,7 +72,23 @@ case "$("$__explorer/os")" in esac ;; devuan) - cat /etc/devuan_version + devuan_version=$(cat /etc/devuan_version) + case ${devuan_version} + in + (*/ceres) + # ceres versions don't have a number, so we decode by codename: + case ${devuan_version} + in + (chimaera/ceres) echo 3.99 ;; + (beowulf/ceres) echo 2.99 ;; + (ascii/ceres) echo 1.99 ;; + (*) exit 1 + esac + ;; + (*) + echo "${devuan_version}" + ;; + esac ;; fedora) cat /etc/fedora-release @@ -68,12 +97,20 @@ case "$("$__explorer/os")" in cat /etc/gentoo-release ;; macosx) - sw_vers -productVersion + # NOTE: Legacy versions (< 10.3) do not support options + sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - freebsd-version + if command -v freebsd-version >/dev/null 2>&1 + then + # get userland version + freebsd-version -u + else + # fallback to kernel release for FreeBSD < 10.0 + uname -r + fi ;; *bsd|solaris) uname -r @@ -98,7 +135,20 @@ case "$("$__explorer/os")" in fi ;; ubuntu) - lsb_release -sr + if command -v lsb_release >/dev/null 2>&1 + then + lsb_release -sr + elif test -r /usr/lib/os-release + then + # fallback to /usr/lib/os-release if lsb_release is not present (like + # on minimized Ubuntu installations) + rc_getvar /usr/lib/os-release VERSION_ID + elif test -r /etc/lsb-release + then + # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old + # versions without /usr/lib/os-release. + rc_getvar /etc/lsb-release DISTRIB_RELEASE + fi ;; alpine) cat /etc/alpine-release diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index bc47d8de..6fcd9212 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,6 +28,7 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 + # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index 38f1bd3c..8ab268c1 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,18 +27,25 @@ else keyid="$__object_id" fi +# From apt-key(8): +# Use of apt-key is deprecated, except for the use of apt-key del in +# maintainer scripts to remove existing keys from the main keyring. +# If such usage of apt-key is desired the additional installation of +# the GNU Privacy Guard suite (packaged in gnupg) is required. +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" + then echo present + else echo absent + fi + exit +fi + keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" -if [ -d "$keydir" ] +if [ -f "$keyfile" ] then - if [ -f "$keyfile" ] - then echo present - else echo absent - fi -else - # fallback to deprecated apt-key - apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ - && echo present \ - || echo absent + echo present + exit fi +echo absent diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 0c96ff67..17dc9bfc 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -25,11 +25,7 @@ else fi state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" - -if [ "$state_should" = "$state_is" ]; then - # nothing to do - exit 0 -fi +method="$(cat "$__object/key_method")" keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" @@ -37,30 +33,18 @@ keyfile="$keydir/$__object_id.gpg" case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - - if [ -f "$__object/parameter/uri" ]; then - uri="$(cat "$__object/parameter/uri")" - - if [ -d "$keydir" ]; then - cat << EOF - -curl -s -L \\ - -o "$keyfile" \\ - "$uri" - -key="\$( cat "$keyfile" )" - -if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' -then - echo "\$key" | gpg --dearmor > "$keyfile" -fi - -EOF - else - # fallback to deprecated apt-key - echo "curl -s -L '$uri' | apt-key add -" + # Using __download or __file as key source + # Propagate messages if needed + if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then + if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then + echo "added '$keyid'" >> "$__messages_out" fi - elif [ -d "$keydir" ]; then + exit 0 + elif [ "${state_is}" = "present" ]; then + exit 0 + fi + # Using key servers to fetch the key + if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then # we need to kill gpg after 30 seconds, because gpg # can get stuck if keyserver is not responding. # exporting env var and not exit 1, @@ -100,13 +84,16 @@ EOF echo "added '$keyid'" >> "$__messages_out" ;; absent) - if [ -f "$keyfile" ]; then - echo "rm '$keyfile'" - else + # Removal for keys added from a keyserver without this flag + # is done in the manifest + if [ "$state_is" != "absent" ] && \ + [ -f "$__object/parameter/use-deprecated-apt-key" ]; then # fallback to deprecated apt-key echo "apt-key del \"$keyid\"" + echo "removed '$keyid'" >> "$__messages_out" + # Propagate messages if needed + elif grep -Eq "^__file$keyfile" "$__messages_in"; then + echo "removed '$keyid'" >> "$__messages_out" fi - - echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index 234bc715..e35eaa0f 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -10,6 +10,14 @@ DESCRIPTION ----------- Manages the list of keys used by apt to authenticate packages. +This is done by placing the requested key in a file named +``$__object_id.gpg`` in the ``keydir`` directory. + +This is supported by modern releases of Debian-based distributions. + +In order of preference, exactly one of: ``source``, ``uri`` or ``keyid`` +must be specified. + REQUIRED PARAMETERS ------------------- @@ -18,21 +26,49 @@ None. OPTIONAL PARAMETERS ------------------- +keydir + keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is + enabled system-wide by default. + +source + path to a file containing the GPG key of the repository. + Using this is recommended as it ensures that the manifest/type manintainer + has validated the key. + If ``-``, the GPG key is read from the type's stdin. + state 'present' or 'absent'. Defaults to 'present' +uri + the URI from which to download the key. + It is highly recommended that you only use protocols with TLS like HTTPS. + This uses ``__download`` but does not use checksums, if you want to ensure + that the key doesn't change, you are better off downloading it and using + ``--source``. + + +DEPRECATED OPTIONAL PARAMETERS +------------------------------ keyid - the id of the key to add. Defaults to __object_id + the id of the key to download from the ``keyserver``. + This is to be used in absence of ``--source`` and ``--uri`` or together + with ``--use-deprecated-apt-key`` for key removal. + Defaults to ``$__object_id``. keyserver - the keyserver from which to fetch the key. If omitted the default set - in ./parameter/default/keyserver is used. + the keyserver from which to fetch the key. + Defaults to ``pool.sks-keyservers.net``. -keydir - key save location, defaults to ``/etc/apt/trusted.pgp.d`` -uri - the URI from which to download the key +DEPRECATED BOOLEAN PARAMETERS +----------------------------- +use-deprecated-apt-key + ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04. + You can use this parameter to force usage of ``apt-key(8)``. + Please only use this parameter to *remove* keys from the keyring, + in order to prepare for removal of ``apt-key``. + Adding keys should be done without this parameter. + This parameter will be removed when Debian 11 stops being supported. EXAMPLES @@ -40,33 +76,39 @@ EXAMPLES .. code-block:: sh - # Add Ubuntu Archive Automatic Signing Key - __apt_key 437D05B5 - # Same thing - __apt_key 437D05B5 --state present - # Get rid of it - __apt_key 437D05B5 --state absent + # add a key that has been verified by a type maintainer + __apt_key jitsi_meet_2021 \ + --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg - # same thing with human readable name and explicit keyid - __apt_key UbuntuArchiveKey --keyid 437D05B5 + # remove an old, deprecated or expired key + __apt_key jitsi_meet_2016 --state absent - # same thing with other keyserver - __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com + # Get rid of a key that might have been added to + # /etc/apt/trusted.gpg with apt-key + __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent - # download key from the internet - __apt_key rabbitmq \ - --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc + # add a key that we define in-line + __apt_key jitsi_meet_2021 --source '-' < Ander Punnar +Evilham COPYING ------- -Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can +Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest index 010357cd..889a764a 100755 --- a/cdist/conf/type/__apt_key/manifest +++ b/cdist/conf/type/__apt_key/manifest @@ -2,7 +2,105 @@ __package gnupg -if [ -f "$__object/parameter/uri" ] -then __package curl -else __package dirmngr +state_should="$(cat "${__object}/parameter/state")" + +incompatible_args() +{ + cat >> /dev/stderr <<-EOF + This type does not support --${1} and --${method} simultaneously. + EOF + exit 1 +} + +if [ -f "${__object}/parameter/source" ]; then + method="source" + src="$(cat "${__object}/parameter/source")" + if [ "${src}" = "-" ]; then + src="${__object}/stdin" + fi +fi +if [ -f "${__object}/parameter/uri" ]; then + if [ -n "${method}" ]; then + incompatible_args uri + fi + method="uri" + src="$(cat "${__object}/parameter/uri")" +fi +if [ -f "${__object}/parameter/keyid" ]; then + if [ -n "${method}" ]; then + incompatible_args keyid + fi + method="keyid" +fi +# Keep old default +if [ -z "${method}" ]; then + method="keyid" +fi +# Save this for later in gencode-remote +echo "${method}" > "${__object}/key_method" + +# Required remotely (most likely already installed) +__package dirmngr +# We need this in case a key has to be dearmor'd +__package gnupg +export require="__package/gnupg" + +if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then + # This is required if apt-key(8) is to be used + if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then + incompatible_args use-deprecated-apt-key + fi +else + if [ "${state_should}" = "absent" ] && \ + [ -f "${__object}/parameter/keyid" ]; then + cat >> /dev/stderr < + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest new file mode 100755 index 00000000..e72a8fdd --- /dev/null +++ b/cdist/conf/type/__apt_pin/manifest @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +name="$__object_id" + +os=$(cat "$__global/explorer/os") +state="$(cat "$__object/parameter/state")" + +if [ -f "$__object/parameter/package" ]; then + package="$(cat "$__object/parameter/package")" +else + package=$name +fi + +distribution="$(cat "$__object/parameter/distribution")" +priority="$(cat "$__object/parameter/priority")" + + +case "$os" in + debian|ubuntu|devuan) + ;; + *) + printf "This type is specific to Debian and it's derivatives" >&2 + exit 1 + ;; +esac + +case $distribution in + stable|testing|unstable|experimental) + pin="release a=$distribution" + ;; + *) + pin="release n=$distribution" + ;; +esac + + +__file "/etc/apt/preferences.d/$name" \ + --owner root --group root --mode 0644 \ + --state "$state" \ + --source - << EOF +Package: $package +Pin: $pin +Pin-Priority: $priority +EOF diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional new file mode 100644 index 00000000..52f01fd2 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -0,0 +1,2 @@ +state +package diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required new file mode 100644 index 00000000..4b4e9741 --- /dev/null +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -0,0 +1,2 @@ +distribution +priority diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state new file mode 100644 index 00000000..f8a3f6c8 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/explorer/state @@ -0,0 +1,142 @@ +#!/bin/sh -e +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Determine current debconf selections' state. +# Prints one of: +# present: all selections are already set as they should. +# different: one or more of the selections have a different value. +# absent: one or more of the selections are not (currently) defined. +# + +test -x /usr/bin/perl || { + # cannot find perl (no perl ~ no debconf) + echo 'absent' + exit 0 +} + +linesfile="${__object:?}/parameter/line" +test -s "${linesfile}" || { + if test -s "${__object:?}/parameter/file" + then + echo absent + else + echo present + fi + exit 0 +} + +# assert __type_explorer is set (because it is used by the Perl script) +: "${__type_explorer:?}" + +/usr/bin/perl -- - "${linesfile}" <<'EOF' +use strict; +use warnings "all"; + +use Fcntl qw(:DEFAULT :flock); + +use Debconf::Db; +use Debconf::Question; + +# Extract @known... arrays from debconf-set-selections +# These values are required to distinguish flags and values in the given lines. +# DC: I couldn't think of a more ugly solution to the problem… +my @knownflags; +my @knowntypes; +my $debconf_set_selections = '/usr/bin/debconf-set-selections'; +if (-e $debconf_set_selections) { + my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; + eval `sed -n '$sed_known' '$debconf_set_selections'`; +} + +sub mungeline ($) { + my $line = shift; + chomp $line; + $line =~ s/\r$//; + return $line; +} + +sub fatal { printf STDERR @_; exit 1; } + +my $state = 'present'; + +sub state { + my $new = shift; + if ($state eq 'present' + or ($state eq 'different' and $new eq 'absent')) { + $state = $new; + } +} + + +# Load Debconf DB but manually lock on the state explorer script, +# because Debconf aborts immediately if executed concurrently. +# This is not really an ideal solution because the Debconf DB could be locked by +# another process (e.g. apt-get), but no way to achieve this could be found. +# If you know how to, please provide a patch. +my $lockfile = "%ENV{'__type_explorer'}/state"; +if (open my $lock_fh, '+<', $lockfile) { + flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; +} +{ + Debconf::Db->load(readonly => 'true'); +} + + +while (<>) { + # Read and process lines (taken from debconf-set-selections) + $_ = mungeline($_); + while (/\\$/ && ! eof) { + s/\\$//; + $_ .= mungeline(<>); + } + next if /^\s*$/ || /^\s*\#/; + + my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ + or fatal "invalid line: %s\n", $_; + $content = '' unless defined $content; + + + # Compare is and should state + my $q = Debconf::Question->get($label); + + unless (defined $q) { + # probably a preseed + state 'absent'; + next; + } + + if (grep { $_ eq $q->type } @knownflags) { + # This line wants to set a flag, presumably. + if ($q->flag($q->type) ne $content) { + state 'different'; + } + } else { + # Otherwise, it's probably a value… + if ($q->value ne $content) { + state 'different'; + } + + unless (grep { $_ eq $owner } (split /, /, $q->owners)) { + state 'different'; + } + } +} + +printf "%s\n", $state; +EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index e99aef40..9ba28f09 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -17,16 +18,37 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# Setup selections -# -filename="$(cat "$__object/parameter/file")" - -if [ "$filename" = "-" ]; then - filename="$__object/stdin" +if test -f "${__object:?}/parameter/line" +then + filename="${__object:?}/parameter/line" +elif test -s "${__object:?}/parameter/file" +then + filename=$(cat "${__object:?}/parameter/file") + if test "${filename}" = '-' + then + filename="${__object:?}/stdin" + fi +else + printf 'Neither --line nor --file set.\n' >&2 + exit 1 fi -echo "debconf-set-selections << __file-eof" -cat "$filename" -echo "__file-eof" +# setting no lines makes no sense +test -s "${filename}" || exit 0 + +state_is=$(cat "${__object:?}/explorer/state") + +if test "${state_is}" != 'present' +then + cat <<-CODE + debconf-set-selections <<'EOF' + $(cat "${filename}") + EOF + CODE + + awk ' + { + printf "set %s %s %s %s\n", $1, $2, $3, $4 + }' "${filename}" >>"${__messages_out:?}" +fi diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index 58c25b81..fd0040ae 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,15 +8,33 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems debconf-set-selections(1) can be used +On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- +cf. ``--line``. + + +OPTIONAL PARAMETERS +------------------- file - Use the given filename as input for debconf-set-selections(1) - If filename is "-", read from stdin. + Use the given filename as input for :strong:`debconf-set-selections`\ (1) + If filename is ``-``, read from stdin. + + **This parameter is deprecated, because it doesn't work with state detection.** +line + A line in :strong:`debconf-set-selections`\ (1) compatible format. + This parameter can be used multiple times to set multiple options. + + (This parameter is actually required, but marked optional because the + deprecated ``--file`` is still accepted.) + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -24,30 +42,29 @@ EXAMPLES .. code-block:: sh - # Setup configuration for nslcd - __debconf_set_selections nslcd --file /path/to/file + # Setup gitolite's gituser + __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' - # Setup configuration for nslcd from another type - __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" - - __debconf_set_selections nslcd --file - << eof - gitolite gitolite/gituser string git - eof + # Setup configuration for nslcd from a file. + # NB: Multiple lines can be passed to --line, although this can be considered a hack. + __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" SEE ALSO -------- -:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`cdist-type__update_alternatives`\ (7) +- :strong:`debconf-set-selections`\ (1) AUTHORS ------- -Nico Schottelius +| Nico Schottelius +| Dennis Camera COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__debconf_set_selections/manifest similarity index 56% rename from cdist/conf/type/__rsync/gencode-remote rename to cdist/conf/type/__debconf_set_selections/manifest index 074246af..0f4fb2e2 100755 --- a/cdist/conf/type/__rsync/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,20 +18,4 @@ # along with cdist. If not, see . # -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") -else - destination="/$__object_id" -fi - -ownergroup="" -if [ -f "$__object/parameter/owner" ]; then - ownergroup=$(cat "$__object/parameter/owner") -fi -if [ -f "$__object/parameter/group" ]; then - ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" -fi - -if [ "$ownergroup" ]; then - echo chown -R "$ownergroup" "$destination" -fi +__package_apt debconf diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__debconf_set_selections/parameter/required b/cdist/conf/type/__debconf_set_selections/parameter/deprecated similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/required rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple new file mode 100644 index 00000000..a999a0c2 --- /dev/null +++ b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple @@ -0,0 +1 @@ +line diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd deleted file mode 100755 index e3e35b45..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cmd="$( cat "$__object/parameter/cmd-get" )" - -elif command -v curl > /dev/null -then - cmd="curl -L -o - '%s'" - -elif command -v fetch > /dev/null -then - cmd="fetch -o - '%s'" - -else - cmd="wget -O - '%s'" -fi - -echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get new file mode 100755 index 00000000..9f1cd59c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_get @@ -0,0 +1,16 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cat "$__object/parameter/cmd-get" +elif + command -v curl > /dev/null +then + echo "curl -sSL -o - '%s'" +elif + command -v fetch > /dev/null +then + echo "fetch -o - '%s'" +else + echo "wget -O - '%s'" +fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum new file mode 100755 index 00000000..84df663c --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd_sum @@ -0,0 +1,82 @@ +#!/bin/sh -e + +if [ ! -f "$__object/parameter/sum" ] +then + exit 0 +fi + +if [ -f "$__object/parameter/cmd-sum" ] +then + cat "$__object/parameter/cmd-sum" + exit 0 +fi + +sum_should="$( cat "$__object/parameter/sum" )" + +if echo "$sum_should" | grep -Fq ':' +then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" +else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi +fi + +os="$( "$__explorer/os" )" + +case "$sum_hash" in + cksum) + echo "cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + case "$os" in + freebsd) + echo "md5 -q %s" + ;; + *) + echo "md5sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha1) + case "$os" in + freebsd) + echo "sha1 -q %s" + ;; + *) + echo "sha1sum %s | awk '{print \$1}'" + ;; + esac + ;; + sha256) + case "$os" in + freebsd) + echo "sha256 -q %s" + ;; + *) + echo "sha256sum %s | awk '{print \$1}'" + ;; + esac + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 00362545..8c5d5ce1 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,6 +1,11 @@ #!/bin/sh -e -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ ! -f "$dst" ] then @@ -8,59 +13,27 @@ then exit 0 fi +if [ ! -f "$__object/parameter/sum" ] +then + echo 'present' + exit 0 +fi + sum_should="$( cat "$__object/parameter/sum" )" -if [ -f "$__object/parameter/cmd-sum" ] +if echo "$sum_should" | grep -Fq ':' then - # shellcheck disable=SC2059 - sum_is="$( eval "$( printf \ - "$( cat "$__object/parameter/cmd-sum" )" \ - "$dst" )" )" -else - os="$( "$__explorer/os" )" - - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" - - elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' - then - case "$os" in - freebsd) - sum_is="md5:$( md5 -q "$dst" )" - ;; - *) - sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' - then - case "$os" in - freebsd) - sum_is="sha1:$( sha1 -q "$dst" )" - ;; - *) - sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" - ;; - esac - - elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' - then - case "$os" in - freebsd) - sum_is="sha256:$( sha256 -q "$dst" )" - ;; - *) - sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" - ;; - esac - fi + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" fi +sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" + +# shellcheck disable=SC2059 +sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" + if [ -z "$sum_is" ] then - echo 'no checksum from target' >&2 + echo 'existing destination checksum failed' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index 571d2c3c..d1b0d0d5 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,34 +11,133 @@ fi url="$( cat "$__object/parameter/url" )" -tmp="$( mktemp )" - -dst="/$__object_id" +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" +else + dst="/$__object_id" +fi if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - elif command -v curl > /dev/null then - cmd="curl -L -o - '%s'" + cmd="curl -sSL -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + else - echo 'no usable locally installed utility for downloading' >&2 + echo 'local download failed, no usable utility' >&2 exit 1 fi -printf "$cmd > %s\n" \ - "$url" \ - "$tmp" +echo "download_tmp=\"\$( mktemp )\"" + +# shellcheck disable=SC2059 +printf "$cmd > \"\$download_tmp\"\n" "$url" + +if [ -f "$__object/parameter/sum" ] +then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + if echo "$sum_should" | grep -Fq ':' + then + sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" + + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + else + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_hash='cksum' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' + then + sum_hash='md5' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' + then + sum_hash='sha1' + elif + echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' + then + sum_hash='sha256' + else + echo 'hash format detection failed' >&2 + exit 1 + fi + fi + + case "$sum_hash" in + cksum) + local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" + ;; + md5) + if command -v md5 > /dev/null + then + local_cmd_sum="md5 -q %s" + elif + command -v md5sum > /dev/null + then + local_cmd_sum="md5sum %s | awk '{print \$1}'" + fi + ;; + sha1) + if command -v sha1 > /dev/null + then + local_cmd_sum="sha1 -q %s" + elif + command -v sha1sum > /dev/null + then + local_cmd_sum="sha1sum %s | awk '{print \$1}'" + fi + ;; + sha256) + if command -v sha256 > /dev/null + then + local_cmd_sum="sha256 -q %s" + elif + command -v sha256sum > /dev/null + then + local_cmd_sum="sha256sum %s | awk '{print \$1}'" + fi + ;; + *) + # we arrive here only if --sum is given with unknown format prefix + echo "unknown hash format: $sum_hash" >&2 + exit 1 + ;; + esac + + if [ -z "$local_cmd_sum" ] + then + echo 'local checksum verification failed, no usable utility' >&2 + exit 1 + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'local download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' +fi if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -47,12 +146,10 @@ else target_host="$__target_host" fi -printf '%s %s %s:%s\n' \ +# shellcheck disable=SC2016 +printf '%s "$download_tmp" %s:%s\n' \ "$__remote_copy" \ - "$tmp" \ "$target_host" \ "$dst" -echo "rm -f '$tmp'" - -echo 'downloaded' > "$__messages_out" +echo "rm -f \"\$download_tmp\"" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index 029a0801..e49bcec3 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,17 +6,51 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd="$( cat "$__object/explorer/remote_cmd" )" + cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" url="$( cat "$__object/parameter/url" )" - dst="/$__object_id" + if [ -f "$__object/parameter/destination" ] + then + dst="$( cat "$__object/parameter/destination" )" + else + dst="/$__object_id" + fi - printf "$cmd > %s\n" \ - "$url" \ - "$dst" + echo "download_tmp=\"\$( mktemp )\"" - echo 'downloaded' > "$__messages_out" + # shellcheck disable=SC2059 + printf "$cmd_get > \"\$download_tmp\"\n" "$url" + + if [ -f "$__object/parameter/sum" ] + then + sum_should="$( cat "$__object/parameter/sum" )" + + if [ -f "$__object/parameter/cmd-sum" ] + then + remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" + else + remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" + + if echo "$sum_should" | grep -Fq ':' + then + sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + fi + fi + + # shellcheck disable=SC2059 + echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" + + echo "if [ \"\$sum_is\" != '$sum_should' ]; then" + + echo "echo 'remote download checksum mismatch' >&2" + + echo "rm -f \"\$download_tmp\"" + + echo 'exit 1; fi' + fi + + echo "mv \"\$download_tmp\" '$dst'" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index 54503470..c16510a9 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,10 +8,7 @@ cdist-type__download - Download a file DESCRIPTION ----------- -Destination (``$__object_id``) in target host must be persistent storage -in order to calculate checksum and decide if file must be (re-)downloaded. - -By default type will try to use ``wget``, ``curl`` or ``fetch``. +By default type will try to use ``curl``, ``fetch`` or ``wget``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -19,23 +16,40 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). +To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. + REQUIRED PARAMETERS ------------------- url File's URL. -sum - Checksum of file going to be downloaded. - By default output of ``cksum`` without filename is expected. - Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. - OPTIONAL PARAMETERS ------------------- +destination + Downloaded file's destination in target. If unset, ``$__object_id`` is used. + +sum + Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. + + Type tries to detect hash format with regexes, but prefixes + ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. + + Checksum have two purposes - state check and post-download verification. + In state check, if destination checksum mismatches, then content of URL + will be downloaded to temporary file. If downloaded temporary file's + checksum matches, then it will be moved to destination (overwritten). + + For local downloads it is expected that usable utilities for checksum + calculation exist in the system. + download - If ``local`` (default), then download file to local storage and copy - it to target host. If ``remote``, then download happens in target. + If ``local`` (default), then file is downloaded to local storage and copied + to target host. If ``remote``, then download happens in target. + + For local downloads it is expected that usable utilities for downloading + exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. cmd-get Command used for downloading. @@ -65,7 +79,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum md5:46da3021ca9eace277115ec9106c5b46 + --sum 46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ @@ -81,7 +95,7 @@ Ander Punnar COPYING ------- -Copyright \(C) 2020 Ander Punnar. You can redistribute it +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 7ec8d86d..3d4c498b 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd" +if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index 838e2fbf..e809ef78 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,4 +1,6 @@ cmd-get cmd-sum +destination download onchange +sum diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required index 6ea4c38f..96cdd3b9 100644 --- a/cdist/conf/type/__download/parameter/required +++ b/cdist/conf/type/__download/parameter/required @@ -1,2 +1 @@ url -sum diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index 9be3c575..d376c09f 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|redhat|suse|gentoo) + alpine|centos|fedora|gentoo|redhat|suse|ubuntu) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 3ddf9656..ab4396b1 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -1,5 +1,24 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id/.git" +destination="/${__object_id:?}/.git" -stat --print "%G" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((group_gid + 1)) -ge 0 +then + group_should=$(cat "${__object:?}/parameter/group") + + if expr "${group_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${group_gid}" + else + if command -v getent > /dev/null + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi + fi +fi diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 4c3cd431..4a4d0d13 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id/.git" +destination="/${__object_id:?}/.git" -stat --print "%U" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((owner_uid + 1)) -ge 0 +then + owner_should=$(cat "${__object:?}/parameter/owner") + + if expr "${owner_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${owner_uid}" + else + printf '%s\n' "$(id -u -n "${owner_uid}")" + fi +fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path deleted file mode 100755 index 3c6076df..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -e - -command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data new file mode 100755 index 00000000..ff62e742 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data @@ -0,0 +1,78 @@ +#!/bin/sh -e +certbot_path="$(command -v certbot 2>/dev/null || true)" +# Defaults +certificate_exists="no" +certificate_is_test="no" + +if [ -n "${certbot_path}" ]; then + # Find python executable that has access to certbot's module + python_path=$(sed -n '1s/^#! *//p' "${certbot_path}") + + # Use a lock for cdist due to certbot not exiting with failure + # or having any flags for concurrent use. + _certbot() { + ${python_path} - 2>/dev/null < "${existing_domains}" + certificate_is_test="$(_explorer_var certificate_is_test)" sort -uo "${requested_domains}" "${requested_domains}" sort -uo "${existing_domains}" "${existing_domains}" diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 1df3574a..6394f629 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,6 +1,6 @@ #!/bin/sh -certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" +certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)" state=$(cat "${__object}/parameter/state") os="$(cat "${__global:?}/explorer/os")" diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 3f88f6bc..ca9aa45a 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,6 +37,7 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 + # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user new file mode 100644 index 00000000..c6582dc4 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/postgres_user @@ -0,0 +1,64 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +os=$("${__explorer:?}/os") + +case ${os} +in + (alpine) + echo 'postgres' + ;; + (centos|rhel|scientific) + echo 'postgres' + ;; + (debian|devuan|ubuntu) + echo 'postgres' + ;; + (freebsd) + test -x /usr/local/etc/rc.d/postgresql || { + printf 'could not find postgresql rc script./n' >&2 + exit 1 + } + pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || { + printf 'postgresql daemon is not running.\n' >&2 + exit 1 + } + pg_pid=$(printf '%s\n' "${pg_status}" \ + | sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p') + + # PostgreSQL < 9.6: pgsql + # PostgreSQL >= 9.6: postgres + ps -o user -p "${pg_pid}" | sed -n '2p' + ;; + (netbsd) + echo 'pgsql' + ;; + (openbsd) + echo '_postgresql' + ;; + (suse) + echo 'postgres' + ;; + (*) + echo "Unsupported OS: ${os}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state new file mode 100644 index 00000000..4b7b0a43 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/explorer/state @@ -0,0 +1,223 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +postgres_user=$("${__type_explorer:?}/postgres_user") +conf_name=${__object_id:?} + +tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; } + +tobytes() { + # NOTE: This function treats everything as base 2. + # It is not compatible with SI units. + awk 'BEGIN { FS = "\n" } + /TB$/ { $0 = ($0 * 1024) "GB" } + /GB$/ { $0 = ($0 * 1024) "MB" } + /MB$/ { $0 = ($0 * 1024) "kB" } + /kB$/ { $0 = ($0 * 1024) "B" } + /B?$/ { sub(/ *B?$/, "") } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tomillisecs() { + awk 'BEGIN { FS = "\n" } + /d$/ { $0 = ($0 * 24) "h" } + /h$/ { $0 = ($0 * 60) "min" } + /min$/ { $0 = ($0 * 60) "s" } + /[^m]s$/ { $0 = ($0 * 1000) "ms" } + /ms$/ { $0 *= 1 } + ($0*1) == $0 # is number + ' <<-EOF + $1 + EOF +} + +tobool() { + # prints either 'on' or 'off' + case $(tolower "$1") + in + (t|true|y|yes|on|1) + echo 'on' ;; + (f|false|n|no|off|0) + echo 'off' ;; + (*) + printf 'Inavlid bool value: %s\n' "$2" >&2 + return 1 + ;; + esac + return 0 +} + +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" +} + +psql_conf_source() { + # NOTE: SHOW/SET are case-insentitive, so this command should also be. + psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" +} +psql_conf_cmp() ( + IFS='|' read -r lower_name vartype setting unit <<-EOF + $(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')") + EOF + + should_value=$2 + is_value=${setting} + + # The following case contains special cases for special settings. + case ${lower_name} + in + (archive_command) + if test "${setting}" = '(disabled)' + then + # DAFUQ PostgreSQL?! + # PostgreSQL returns (disabled) if the feature is inactive. + # We cannot compare the values unless it is enabled, first. + return 0 + fi + ;; + (archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit) + # Although only 'on', 'off' are documented, PostgreSQL accepts all + # the "likely" variants of "on" and "off". + case $(tolower "${should_value}") + in + (on|off|true|false|yes|no|1|0) + should_value=$(tobool "${should_value}") + ;; + esac + ;; + esac + + case ${vartype} + in + (bool) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported for vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + should_value=$(tobool "${should_value}") + + test "${is_value}" = "${should_value}" + ;; + (enum) + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + # NOTE: All enums that are currently defined are lower case, but + # PostgreSQL also accepts upper case spelling. + should_value=$(tolower "$2") + + test "${is_value}" = "${should_value}" + ;; + (integer) + # split multiples from unit, first (e.g. 8kB -> 8, kB) + case ${unit} + in + ([0-9]*) + multiple=${unit%%[!0-9]*} + unit=${unit##*[0-9 ]} + ;; + (*) multiple=1 ;; + esac + + is_value=$((setting * multiple))${unit} + + if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null + then + # default unit + should_value=$((should_value * multiple))${unit} + fi + + # then, do conversion + # NOTE: these conversions work for integers only! + case ${unit} + in + (B|[kMGT]B) + # bytes + is_bytes=$(tobytes "${is_value}") + should_bytes=$(tobytes "${should_value}") + + test $((is_bytes)) -eq $((should_bytes)) + ;; + (ms|s|min|h|d) + # seconds + is_ms=$(tomillisecs "${is_value}") + should_ms=$(tomillisecs "${should_value}") + + test $((is_ms)) -eq $((should_ms)) + ;; + ('') + # no unit + is_int=${is_value} + should_int=${should_value} + + test $((is_int)) -eq $((should_int)) + ;; + esac + ;; + (real|string) + # NOTE: reals could possibly have units, but currently there none. + + test -z "${unit}" || { + # please fix the explorer if this error occurs. + printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 + exit 1 + } + + test "${is_value}" = "${should_value}" + ;; + esac +) + +psql_exec 'SELECT 1' >/dev/null || { + echo 'Connection to PostgreSQL server failed' >&2 + exit 1 +} + +case $(psql_conf_source "${conf_name}") +in + ('') + printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2 + exit 1 + ;; + (default) + echo absent + ;; + (*) + if ! test -f "${__object:?}/parameter/value" + then + echo present + elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" + then + echo present + else + echo different + fi + ;; +esac diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote new file mode 100755 index 00000000..27651600 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/gencode-remote @@ -0,0 +1,123 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +state_is=$(cat "${__object:?}/explorer/state") +state_should=$(cat "${__object:?}/parameter/state") +postgres_user=$(cat "${__object:?}/explorer/postgres_user") + +conf_name=${__object_id:?} + +if test "${state_is}" = "${state_should}" +then + exit 0 +fi + +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" + then + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" + fi + done + unset _arg + + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} + + +psql_cmd() { + printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" +} + +case ${state_should} +in + (present) + test -n "${__object:?}/parameter/value" || { + echo 'Missing required parameter --value' >&2 + exit 1 + } + + cat <<-EOF + exec 3< "\${__object:?}/parameter/value" + $(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL' + \\set conf_value \`cat <&3\` + ALTER SYSTEM SET ${conf_name} = :'conf_value'; + SELECT pg_reload_conf(); + SQL + exec 3<&- + EOF + ;; + (absent) + psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT" + ;; + (*) + printf 'Invalid --state: %s\n' "${state_should}" >&2 + printf 'Only "present" and "absent" are acceptable.\n' >&2 + exit 1 + ;; +esac + +# Restart PostgreSQL server if required to apply new configuration value +cat <&2 + exit 1 + esac + ;; + (*) + printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2 + exit 1 + esac + ) +fi +EOF diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst new file mode 100644 index 00000000..e035f080 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/man.rst @@ -0,0 +1,60 @@ +cdist-type__postgres_conf(7) +============================ + +NAME +---- +cdist-type__postgres_conf - Alter PostgreSQL configuration + + +DESCRIPTION +----------- +Configure a running PostgreSQL server using ``ALTER SYSTEM``. + + +REQUIRED PARAMETERS +------------------- +value + The value to set (can be omitted if ``--state`` is set to ``absent``). + + +OPTIONAL PARAMETERS +------------------- +state + ``present`` or ``absent``. + Defaults to ``present``. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # set timezone + __postgres_conf timezone --value Europe/Zurich + + # reset maximum number of concurrent connections to default (normally 100) + __postgres_conf max_connections --state absent + + +SEE ALSO +-------- +None. + + +AUTHORS +------- +Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch) +Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch) + + +COPYING +------- +Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch). +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional new file mode 100644 index 00000000..d0460d86 --- /dev/null +++ b/cdist/conf/type/__postgres_conf/parameter/optional @@ -0,0 +1,2 @@ +state +value diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_database/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index d68d4120..6a25df86 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -1,6 +1,7 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,25 +19,18 @@ # along with cdist. If not, see . # -case "$("${__explorer}/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +postgres_user=$("${__type_explorer:?}/postgres_user") +dbname=${__object_id:?} -name="$__object_id" +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_exec() { + su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")" +} -if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" +if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 0f11cff4..7d7d6fa2 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,6 +1,7 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,60 +19,63 @@ # along with cdist. If not, see . # -case "$(cat "${__global}/explorer/os")" -in - netbsd) - postgres_user='pgsql' - ;; - openbsd) - postgres_user='_postgresql' - ;; - *) - postgres_user='postgres' - ;; -esac +quote() { + for _arg + do + shift + if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" + then + # needs quoting + set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" + else + set -- "$@" "${_arg}" + fi + done + unset _arg + # NOTE: Use printf because POSIX echo interprets escape sequences + printf '%s' "$*" +} -name="$__object_id" -state_should="$(cat "$__object/parameter/state")" -state_is="$(cat "$__object/explorer/state")" +postgres_user=$(cat "${__object:?}/explorer/postgres_user") -if [ "$state_should" != "$state_is" ]; then - case "$state_should" in - present) - owner="" - if [ -f "$__object/parameter/owner" ]; then - owner="-O \"$(cat "$__object/parameter/owner")\"" - fi +dbname=${__object_id:?} +state_should=$(cat "${__object:?}/parameter/state") +state_is=$(cat "${__object:?}/explorer/state") - template="" - if [ -f "$__object/parameter/template" ]; then - template="--template \"$(cat "$__object/parameter/template")\"" - fi - - encoding="" - if [ -f "$__object/parameter/encoding" ]; then - encoding="--encoding \"$(cat "$__object/parameter/encoding")\"" - fi - - lc_collate="" - if [ -f "$__object/parameter/lc-collate" ]; then - lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\"" - fi - - lc_ctype="" - if [ -f "$__object/parameter/lc-ctype" ]; then - lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\"" - fi - - cat << EOF -su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype" -EOF - ;; - absent) - cat << EOF -su - '$postgres_user' -c "dropdb \"$name\"" -EOF - ;; - esac +if test "${state_should}" = "$state_is" +then + exit 0 fi + +case ${state_should} +in + (present) + set -- + + while read -r param_name opt + do + if test -f "${__object:?}/parameter/${param_name}" + then + set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")" + fi + done <<-'EOF' + owner -O + template --template + encoding --encoding + lc_collate --lc-collate + lc_ctype --lc-ctype + EOF + + set -- "$@" "${dbname}" + + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")") + EOF + ;; + (absent) + cat <<-EOF + su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")") + EOF + ;; +esac diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__postgres_extension/explorer/state new file mode 100644 index 00000000..9d156be7 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/explorer/state @@ -0,0 +1,41 @@ +#!/bin/sh -e +# -*- mode: sh; indent-tabs-mode: t -*- +# +# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# Prints "present" if the extension is currently installed. +# "absent" otherwise. + +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } + +postgres_user=$("${__type_explorer:?}/postgres_user") + +IFS=: read -r dbname extname <. # -case "$(cat "${__global}/explorer/os")" +postgres_user=$(cat "${__object:?}/explorer/postgres_user") + +quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } +psql_cmd() { + printf 'su - %s -c %s\n' \ + "$(quote "${postgres_user}")" \ + "$(quote psql "$(quote "$1")" -c "$(quote "$2")")" +} + + +IFS=: read -r dbname extname <&2 + exit 1 + ;; esac diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 79645b2b..442239f6 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -3,32 +3,36 @@ cdist-type__postgres_extension(7) NAME ---- -cdist-type__postgres_extension - manage postgres extensions +cdist-type__postgres_extension - Manage PostgreSQL extensions DESCRIPTION ----------- -This cdist type allows you to create or drop postgres extensions. +This cdist type allows you to manage PostgreSQL extensions. -The object you need to pass to __postgres_extension consists of -the database name and the extension name joined by a colon in the -following form: - -.. code-block:: sh - - dbname:extension - -f.ex. +The ``__object_id`` to pass to ``__postgres_extension`` is of the form +``dbname:extension``, e.g.: .. code-block:: sh rails_test:unaccent +**CAUTION!** Be careful when installing extensions from (untrusted) third-party +sources: + + | Installing an extension as superuser requires trusting that the extension's + author wrote the extension installation script in a secure fashion. It is + not terribly difficult for a malicious user to create trojan-horse objects + that will compromise later execution of a carelessly-written extension + script, allowing that user to acquire superuser privileges. + | – ``_ + + OPTIONAL PARAMETERS ------------------- state - either "present" or "absent", defaults to "present" + either ``present`` or ``absent``, defaults to ``present``. EXAMPLES @@ -36,24 +40,29 @@ EXAMPLES .. code-block:: sh - __postgres_extension rails_test:unaccent - __postgres_extension --present rails_test:unaccent - __postgres_extension --absent rails_test:unaccent + # Install extension unaccent into database rails_test + __postgres_extension rails_test:unaccent + + # Drop extension unaccent from database fails_test + __postgres_extension rails_test:unaccent --state absent SEE ALSO -------- -:strong:`cdist-type__postgre_database`\ (7) +- :strong:`cdist-type__postgres_database`\ (7) +- PostgreSQL "CREATE EXTENSION" documentation at: + ``_. -Postgres "Create Extension" documentation at: . -AUTHOR +AUTHORS ------- -Tomas Pospisek +| Tomas Pospisek +| Dennis Camera + COPYING ------- -Copyright \(C) 2014 Tomas Pospisek. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera. +You can redistribute it and/or modify it under the terms of the GNU General +Public License as published by the Free Software Foundation, either version 3 of +the License, or (at your option) any later version. diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user new file mode 120000 index 00000000..714e7237 --- /dev/null +++ b/cdist/conf/type/__postgres_role/explorer/postgres_user @@ -0,0 +1 @@ +../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 34069de9..822816c1 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -19,19 +19,7 @@ # along with cdist. If not, see . # -case $("${__explorer:?}/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - +postgres_user=$("${__type_explorer:?}/postgres_user") rolename=${__object_id:?} @@ -55,8 +43,7 @@ role_properties=$( BEGIN { RS = "\036"; FS = "\034" } /^\([0-9]+ rows?\)/ { exit } NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } - NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i } - ' + NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }' ) if test -n "${role_properties}" @@ -90,12 +77,10 @@ then # Check password passwd_stored=$( psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ - | awk 'BEGIN { RS = "\036" } NR == 2' - printf . - ) - passwd_stored=${passwd_stored%?.} + | awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }') + passwd_stored=${passwd_stored%.} - if test -f "${__object:?}/parameter/password" + if test -s "${__object:?}/parameter/password" then passwd_should=$(cat "${__object:?}/parameter/password"; printf .) fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index d7631fbd..4cb78330 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -28,20 +28,7 @@ quote() { fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" } -case $(cat "${__global:?}/explorer/os") -in - (netbsd) - postgres_user='pgsql' - ;; - (openbsd) - postgres_user='_postgresql' - ;; - (*) - postgres_user='postgres' - ;; -esac - - +postgres_user=$(cat "${__object:?}/explorer/postgres_user") rolename=${__object_id:?} state_is=$(cat "${__object:?}/explorer/state") state_should=$(cat "${__object:?}/parameter/state") @@ -59,7 +46,7 @@ psql_query() { psql_set_password() { # NOTE: Always make sure that the password does not end up in psql_history! - # NOTE: Never set an empty string as the password, because they can be + # NOTE: Never set an empty string as the password, because it can be # interpreted differently by different tooling. if test -s "${__object:?}/parameter/password" then diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index a655bda7..922ce3df 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -1,5 +1,24 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id" +destination="/${__object_id:?}" -stat --print "%G" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((group_gid + 1)) -ge 0 +then + group_should=$(cat "${__object:?}/parameter/group") + + if expr "${group_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${group_gid}" + else + if command -v getent >/dev/null 2>&1 + then + getent group "${group_gid}" | cut -d : -f 1 + else + awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group + fi + fi +fi diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index 8b3c7f8e..ebec751f 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -1,5 +1,19 @@ -#!/bin/sh +#!/bin/sh -e -destination="/$__object_id" +destination="/${__object_id:?}" -stat --print "%U" "${destination}" 2>/dev/null || exit 0 +# shellcheck disable=SC2012 +owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') + +# NOTE: +1 because $((notanum)) prints 0. +if test $((owner_uid + 1)) -ge 0 +then + owner_should=$(cat "${__object:?}/parameter/owner") + + if expr "${owner_should}" : '[0-9]*$' >/dev/null + then + printf '%u\n' "${owner_uid}" + else + printf '%s\n' "$(id -u -n "${owner_uid}")" + fi +fi diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst index 8085ff12..e2e4a1e6 100644 --- a/cdist/conf/type/__pyvenv/man.rst +++ b/cdist/conf/type/__pyvenv/man.rst @@ -61,7 +61,7 @@ EXAMPLES __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4 # Create python virtualenv for user foo. - __pyvenv /home/foo/fooenv --group foo --user foo + __pyvenv /home/foo/fooenv --group foo --owner foo # Create python virtualenv with specific parameters. __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages" diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e36ded2f..e9f3c131 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,39 +1,104 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -source=$(cat "$__object/parameter/source") -remote_user=$(cat "$__object/parameter/remote-user") +if ! command -v rsync > /dev/null +then + echo 'rsync is missing in local machine' >&2 + exit 1 +fi -if [ -f "$__object/parameter/destination" ]; then - destination=$(cat "$__object/parameter/destination") +src="$( cat "$__object/parameter/source" )" + +if [ ! -e "$src" ] +then + echo "$src not found" >&2 + exit 1 +fi + +if [ -f "$__object/parameter/destination" ] +then + dst="$( cat "$__object/parameter/destination" )" else - destination="/$__object_id" + dst="/$__object_id" fi -set -- -if [ -f "$__object/parameter/rsync-opts" ]; then - while read -r opts; do - set -- "$@" "--$opts" - done < "$__object/parameter/rsync-opts" +# if source is directory, then make sure that +# source and destination are ending with slash, +# because this is what you almost always want when +# rsyncing two directories. + +if [ -d "$src" ] +then + if ! echo "$src" | grep -Eq '/$' + then + src="$src/" + fi + + if ! echo "$dst" | grep -Eq '/$' + then + dst="$dst/" + fi fi -echo rsync -a \ - --no-owner --no-group \ - -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" +remote_user="$( cat "$__object/parameter/remote-user" )" + +options="$( cat "$__object/parameter/options" )" + +if [ -f "$__object/parameter/option" ] +then + while read -r l + do + # there's a limitation in argparse: value can't begin with '-'. + # to workaround this, let's prefix opts with '\' in manifest and remove here. + # read more about argparse issue: https://bugs.python.org/issue9334 + + options="$options $( echo "$l" | sed 's/\\//g' )" + done \ + < "$__object/parameter/option" +fi + +if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] +then + options="$options --chown=" + + if [ -f "$__object/parameter/owner" ] + then + owner="$( cat "$__object/parameter/owner" )" + options="$options$owner" + fi + + if [ -f "$__object/parameter/group" ] + then + group="$( cat "$__object/parameter/group" )" + options="$options:$group" + fi +fi + +if [ -f "$__object/parameter/mode" ] +then + mode="$( cat "$__object/parameter/mode" )" + options="$options --chmod=$mode" +fi + +# IMPORTANT +# +# 1. we first dry-run rsync with change summary to find out +# if there are any changes and code generation is needed. +# 2. normally, to get current state or target host, we run +# such operations in type explorers, but that's not +# possible due to how rsync works. +# 3. redirecting output of dry-run to stderr to ease debugging. +# 4. to understand how that cryptic regex works, please +# open rsync manpage and read about --itemize-changes. + +export RSYNC_RSH="$__remote_exec" + +# shellcheck disable=SC2086 +if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ + | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 +then + exit 0 +fi + +echo "export RSYNC_RSH='$__remote_exec'" + +echo "rsync $options $src $remote_user@$__target_host:$dst" diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 94b06d63..88019c92 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,112 +3,73 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using rsync +cdist-type__rsync - Mirror directories using ``rsync`` DESCRIPTION ----------- -WARNING: This type is of BETA quality: - -- it has not been tested widely -- interfaces *may* change -- if there is a better approach to solve the problem -> the type may even vanish - -If you are fine with these constraints, please read on. - - -This cdist type allows you to mirror local directories to the -target host using rsync. Rsync will be installed in the manifest of the type. -If group or owner are giveng, a recursive chown will be executed on the -target host. - -A slash will be appended to the source directory so that only the contents -of the directory are taken and not the directory name itself. +The purpose of this type is to bring power of ``rsync`` into ``cdist``. REQUIRED PARAMETERS ------------------- source - Where to take files from + Source directory in local machine. + If source is directory, slash (``/``) will be added to source and destination paths. OPTIONAL PARAMETERS ------------------- -group - Group to chgrp to. +destination + Destination directory. Defaults to ``$__object_id``. owner - User to chown to. + Will be passed to ``rsync`` as ``--chown=OWNER``. + Read ``rsync(1)`` for more details. -destination - Use this as the base destination instead of the object id +group + Will be passed to ``rsync`` as ``--chown=:GROUP``. + Read ``rsync(1)`` for more details. + +mode + Will be passed to ``rsync`` as ``--chmod=MODE``. + Read ``rsync(1)`` for more details. + +options + Defaults to ``--recursive --links --perms --times``. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. remote-user - Use this user instead of the default "root" for rsync operations. + Defaults to ``root``. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -rsync-opts - Use this option to give rsync options with. - See rsync(1) for available options. - Only "--" options are supported. - Write the options without the beginning "--" - Can be specified multiple times. - - -MESSAGES --------- -NONE +option + Pass additional options to ``rsync``. + See ``rsync(1)`` for all possible options. + Due to `bug in Python's argparse`_, value must be prefixed with ``\``. EXAMPLES -------- - .. code-block:: sh - # You can use any source directory - __rsync /tmp/testdir \ - --source /etc - - # Use source from type - __rsync /etc \ - --source "$__type/files/package" - - # Allow multiple __rsync objects to write to the same dir - __rsync mystuff \ - --destination /usr/local/bin \ - --source "$__type/files/package" - - __rsync otherstuff \ - --destination /usr/local/bin \ - --source "$__type/files/package2" - - # Use rsync option --exclude - __rsync /tmp/testdir \ - --source /etc \ - --rsync-opts exclude=sshd_conf - - # Use rsync with multiple options --exclude --dry-run - __rsync /tmp/testing \ - --source /home/tester \ - --rsync-opts exclude=id_rsa \ - --rsync-opts dry-run - - -SEE ALSO --------- -:strong:`rsync`\ (1) + __rsync /var/www/example.com \ + --owner root \ + --group www-data \ + --mode 'D750,F640' \ + --source "$__files/example.com/www" AUTHORS ------- -Nico Schottelius +Ander Punnar COPYING ------- -Copyright \(C) 2015 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 9bd44c6d..64fa804e 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,21 +1,3 @@ #!/bin/sh -e -# -# 2015 Dominique Roux (dominique.roux4 at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options new file mode 100644 index 00000000..d967b110 --- /dev/null +++ b/cdist/conf/type/__rsync/parameter/default/options @@ -0,0 +1 @@ +--recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index ac2b2390..833e9bbe 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,4 +1,6 @@ destination -owner group +mode +options +owner remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index fdb7cd88..01925a15 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -rsync-opts +option diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group new file mode 100755 index 00000000..a6cb3dfd --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group @@ -0,0 +1,8 @@ +#!/bin/sh -e + +if grep -Eq '^ssl-cert:' /etc/group +then + echo 'present' +else + echo 'absent' +fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state new file mode 100755 index 00000000..cc5aae0b --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/explorer/state @@ -0,0 +1,24 @@ +#!/bin/sh -e + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] +then + echo 'absent' +else + echo 'present' +fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote new file mode 100755 index 00000000..8ffbfad1 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh -e + +state="$( cat "$__object/explorer/state" )" + +if [ "$state" = 'present' ] +then + exit 0 +fi + +if [ -f "$__object/parameter/common-name" ] +then + common_name="$( cat "$__object/parameter/common-name" )" +else + common_name="$__object_id" +fi + +key_path="$( cat "$__object/parameter/key-path" )" + +if echo "$key_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + key_path="$( printf "$key_path" "$__object_id" )" +fi + +cert_path="$( cat "$__object/parameter/cert-path" )" + +if echo "$cert_path" | grep -Fq '%s' +then + # shellcheck disable=SC2059 + cert_path="$( printf "$cert_path" "$__object_id" )" +fi + +key_type="$( cat "$__object/parameter/key-type" )" + +key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" + +case "$key_type" in + rsa:*) + echo "openssl genrsa -out '$key_path' $key_type_arg" + ;; + ec:*) + echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" + ;; +esac + +# shellcheck disable=SC2016 +echo 'csr_path="$( mktemp )"' + +echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" + +echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" + +# shellcheck disable=SC2016 +echo 'rm -f "$csr_path"' + +if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] +then + key_group='ssl-cert' +else + key_group='root' +fi + +echo "chmod 640 '$key_path'" + +echo "chown root '$key_path'" + +echo "chgrp $key_group '$key_path'" + +echo "chmod 644 '$cert_path'" + +echo "chown root '$cert_path'" + +echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst new file mode 100644 index 00000000..b0b0a2e9 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -0,0 +1,61 @@ +cdist-type__snakeoil_cert(7) +============================ + +NAME +---- +cdist-type__snakeoil_cert - Generate self-signed certificate + + +DESCRIPTION +----------- +The purpose of this type is to generate **self-signed** certificate and private key +for **testing purposes**. Certificate will expire in 3650 days. + +Certificate's and key's access bits will be ``644`` and ``640`` respectively. +If target system has ``ssl-cert`` group, then it will be used as key's group. +Use ``require='__snakeoil_cert/...' __file ...`` to override. + + +OPTIONAL PARAMETERS +------------------- +common-name + Defaults to ``$__object_id``. + +key-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/private/%s.pem``. + +key-type + Possible values are ``rsa:$bits`` and ``ec:$name``. + For possible EC names see ``openssl ecparam -list_curves``. + Defaults to ``rsa:2048``. + +cert-path + ``%s`` in path will be replaced with ``$__object_id``. + Defaults to ``/etc/ssl/certs/%s.pem``. + + +EXAMPLES +-------- +.. code-block:: sh + + __snakeoil_cert localhost-rsa \ + --common-name localhost \ + --key-type rsa:4096 + + __snakeoil_cert localhost-ec \ + --common-name localhost \ + --key-type ec:prime256v1 + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the Free +Software Foundation, either version 3 of the License, or (at your option) +any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path new file mode 100644 index 00000000..4bbae089 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path @@ -0,0 +1 @@ +/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path new file mode 100644 index 00000000..86eb9359 --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path @@ -0,0 +1 @@ +/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type new file mode 100644 index 00000000..f13f8ada --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type @@ -0,0 +1 @@ +rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional new file mode 100644 index 00000000..76d08c0a --- /dev/null +++ b/cdist/conf/type/__snakeoil_cert/parameter/optional @@ -0,0 +1,4 @@ +common-name +key-path +key-type +cert-path diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index ccab0afc..aca0f2b9 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -25,6 +25,7 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ] if [ -n "${type_and_key}" ] then file="$(cat "$__object/parameter/file")" + test -e "$file" || exit 0 # get any entries that match the type and key diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index f37aa565..61c77fb9 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -37,9 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" + grep -v -F -x '$line' '$file' >\$tmpfile fi -grep -v -F -x '$line' '$file' > \$tmpfile || true -mv -f "\$tmpfile" "$file" +cat "\$tmpfile" >"$file" DONE } diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index cec25746..9694a64b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,6 +1,7 @@ #!/bin/sh -e # shellcheck disable=SC1090 +# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest index 566bde90..e37afebb 100755 --- a/cdist/conf/type/__sshd_config/manifest +++ b/cdist/conf/type/__sshd_config/manifest @@ -39,7 +39,14 @@ in (freebsd|netbsd|openbsd) # whitelist ;; + (openbmc-phosphor) + # whitelist + # OpenBMC can be configured with dropbear and OpenSSH. + # If dropbear is used, the state explorer will already fail because it + # cannot find the sshd binary. + ;; (*) + : "${__type:?}" # make shellcheck happy printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ "${os}" "${__type##*/}" >&2 printf 'Please contribute an implementation for it if you can.\n' >&2 diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index 34aaca56..bb1619a9 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -update-alternatives --display "$__object_id" 2>/dev/null \ - | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ +| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index 6519e7c2..d1087c75 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/$__object_id" ] + if [ ! -f "$altdir/${__object_id:?}" ] then continue fi - link="$( awk 'NR==2' "$altdir/$__object_id" )" + link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" if [ -n "$link" ] then @@ -31,9 +31,12 @@ do fi done -if [ -z "$link" ] +if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] then - echo "unable to get link for $__object_id" >&2 + # NOTE: ignore error for dry-runs because a package providing the link + # might be managed by another cdist object (which wasn't executed, + # because dry run…). + echo "unable to get link for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index fc304d5d..5cf4fa4b 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,11 +1,15 @@ #!/bin/sh -e -path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ - | awk '/link currently points to/ {print $5}' )" +path_is=$( + LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ + | awk '/link currently points to/ { print $5 }') -if [ -z "$path_is" ] +if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] then - echo "unable to get current path for $__object_id" >&2 + # NOTE: ignore error for dry-runs because a package providing the + # alternative might be managed by another cdist object (which + # wasn't executed, because dry run…). + echo "unable to get current path for ${__object_id:?}" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index 59e015c5..b74a7ee8 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "$__object/parameter/path" )" ] +if [ -f "$( cat "${__object:?}/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e393cdef..e91ea78f 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,37 +18,39 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "$__object/explorer/path_is" )" +path_is="$( cat "${__object:?}/explorer/path_is" )" -path_should="$( cat "$__object/parameter/path" )" +path_should="$( cat "${__object:?}/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] +if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ + && [ -z "${__cdist_dry_run+dry run}" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name="$__object_id" +name=${__object_id:?} -alternatives="$( cat "$__object/explorer/alternatives" )" - -if ! echo "$alternatives" | grep -Fxq "$path_should" +if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" then - if [ ! -f "$__object/parameter/install" ] + if [ -f "${__object:?}/parameter/install" ] then + link="$( cat "${__object:?}/explorer/link" )" + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" + elif [ -z "${__cdist_dry_run+dry run}" ] + then + # NOTE: ignore error for dry-runs because a package providing the link + # to be installed might be managed by another cdist object (which + # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi - - link="$( cat "$__object/explorer/link" )" - - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" diff --git a/cdist/config.py b/cdist/config.py index e84f6f84..638fdf0e 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -190,7 +190,7 @@ class Config: fd.write(sys.stdin.read()) except (IOError, OSError) as e: raise cdist.Error(("Creating tempfile for stdin data " - "failed: %s" % e)) + "failed: {}").format(e)) args.manifest = initial_manifest_temp_path atexit.register(lambda: os.remove(initial_manifest_temp_path)) @@ -273,15 +273,15 @@ class Config: host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) - log.debug("Base root path for target host \"{}\" is \"{}\"".format( - host, host_base_path)) + log.debug("Base root path for target host \"%s\" is \"%s\"", + host, host_base_path) hostcnt += 1 if args.parallel: pargs = (host, host_tags, host_base_path, hostdir, args, True, configuration) - log.trace(("Args for multiprocessing operation " - "for host {}: {}".format(host, pargs))) + log.trace("Args for multiprocessing operation for host %s: %s", + host, pargs) process_args.append(pargs) else: try: @@ -298,10 +298,10 @@ class Config: except cdist.Error: failed_hosts.append(host) elif args.parallel: - log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - log.trace(("Starting multiprocessing Pool for {} " - "parallel host operation".format(args.parallel))) + log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + log.trace("Starting multiprocessing Pool for %d parallel host" + " operation", args.parallel) results = mp_pool_run(cls.onehost, process_args, @@ -396,16 +396,13 @@ class Config: remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) - log.debug("remote_exec for host \"{}\": {}".format( - host, remote_exec)) - log.debug("remote_copy for host \"{}\": {}".format( - host, remote_copy)) + log.debug("remote_exec for host \"%s\": %s", host, remote_exec) + log.debug("remote_copy for host \"%s\": %s", host, remote_copy) family = cls._address_family(args) - log.debug("address family: {}".format(family)) + log.debug("address family: %s", family) target_host = cls.resolve_target_addresses(host, family) - log.debug("target_host for host \"{}\": {}".format( - host, target_host)) + log.debug("target_host for host \"%s\": %s", host, target_host) local = cdist.exec.local.Local( target_host=target_host, @@ -420,6 +417,9 @@ class Config: exec_path=sys.argv[0], save_output_streams=args.save_output_streams) + # Make __global state dir available to custom remote scripts. + os.environ['__global'] = local.base_path + remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, @@ -471,8 +471,8 @@ class Config: """Do what is most often done: deploy & cleanup""" start_time = time.time() - self.log.info("Starting {} run".format( - 'dry' if self.dry_run else 'configuration')) + self.log.info("Starting %s run", + 'dry' if self.dry_run else 'configuration') self._init_files_dirs() @@ -490,9 +490,9 @@ class Config: self._remove_files_dirs() self.local.save_cache(start_time) - self.log.info("Finished {} run in {:.2f} seconds".format( + self.log.info("Finished %s run in %.2f seconds", 'dry' if self.dry_run else 'successful', - time.time() - start_time)) + time.time() - start_time) def cleanup(self): self.log.debug("Running cleanup commands") @@ -516,8 +516,8 @@ class Config: self.local.object_path, self.local.type_path, self.local.object_marker_name): if cdist_object.cdist_type.is_install: - self.log.debug(("Running in config mode, ignoring install " - "object: {0}").format(cdist_object)) + self.log.debug("Running in config mode, ignoring install " + "object: %s", cdist_object) else: yield cdist_object @@ -537,7 +537,7 @@ class Config: objects_changed = False for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -548,7 +548,7 @@ class Config: self.object_prepare(cdist_object) objects_changed = True - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -562,13 +562,13 @@ class Config: return objects_changed def _iterate_once_parallel(self): - self.log.debug("Iteration in parallel mode in {} jobs".format( - self.jobs)) + self.log.debug("Iteration in parallel mode in %d jobs", self.jobs) objects_changed = False cargo = [] for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.has_requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -585,8 +585,8 @@ class Config: self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) self.log.trace("Multiprocessing cargo: %s", cargo) @@ -600,9 +600,8 @@ class Config: "sequentially")) self.explorer.transfer_type_explorers(cargo_types.pop()) else: - self.log.trace(("Starting multiprocessing Pool for {} " - "parallel types explorers transferring".format( - nt))) + self.log.trace("Starting multiprocessing Pool for %d " + "parallel types explorers transferring", nt) args = [ (ct, ) for ct in cargo_types ] @@ -611,8 +610,8 @@ class Config: self.log.trace(("Multiprocessing for parallel transferring " "types' explorers finished")) - self.log.trace(("Starting multiprocessing Pool for {} parallel " - "objects preparation".format(n))) + self.log.trace("Starting multiprocessing Pool for %d parallel " + "objects preparation", n) args = [ (c, False, ) for c in cargo ] @@ -623,12 +622,13 @@ class Config: del cargo[:] for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.has_requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue if cdist_object.state == core.CdistObject.STATE_PREPARED: - if cdist_object.requirements_unfinished( + if cdist_object.has_requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -664,10 +664,10 @@ class Config: self.object_run(chunk[0]) objects_changed = True elif chunk: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for {} " - "parallel object run".format(n))) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + self.log.trace("Starting multiprocessing Pool for %d " + "parallel object run", n) args = [ (c, ) for c in chunk ] @@ -699,20 +699,22 @@ class Config: check for cycles. ''' graph = {} - for cdist_object in self.object_list(): + + def _add_requirements(cdist_object, requirements): obj_name = cdist_object.name if obj_name not in graph: graph[obj_name] = [] + + for requirement in cdist_object.requirements_unfinished( + requirements): + graph[obj_name].append(requirement.name) + + for cdist_object in self.object_list(): if cdist_object.state == cdist_object.STATE_DONE: continue - for requirement in cdist_object.requirements_unfinished( - cdist_object.requirements): - graph[obj_name].append(requirement.name) - - for requirement in cdist_object.requirements_unfinished( - cdist_object.autorequire): - graph[obj_name].append(requirement.name) + _add_requirements(cdist_object, cdist_object.requirements) + _add_requirements(cdist_object, cdist_object.autorequire) return graph_check_cycle(graph) def iterate_until_finished(self): @@ -766,7 +768,7 @@ class Config: raise cdist.UnresolvableRequirementsError( ("The requirements of the following objects could not be " - "resolved:\n%s") % ("\n".join(info_string))) + "resolved:\n{}").format("\n".join(info_string))) def _handle_deprecation(self, cdist_object): cdist_type = cdist_object.cdist_type @@ -791,9 +793,9 @@ class Config: def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) - self.log.verbose("Preparing object {}".format(cdist_object.name)) - self.log.verbose( - "Running manifest and explorers for " + cdist_object.name) + self.log.verbose("Preparing object %s", cdist_object.name) + self.log.verbose("Running manifest and explorers for %s", + cdist_object.name) self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: self.manifest.run_type_manifest(cdist_object) @@ -807,13 +809,13 @@ class Config: def object_run(self, cdist_object): """Run gencode and code for an object""" try: - self.log.verbose("Running object " + cdist_object.name) + self.log.verbose("Running object %s", cdist_object.name) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " - "object: %s"), cdist_object) + "object: {}").format(cdist_object)) # Generate - self.log.debug("Generating code for %s" % (cdist_object.name)) + self.log.debug("Generating code for %s", cdist_object.name) cdist_object.code_local = self.code.run_gencode_local(cdist_object) cdist_object.code_remote = self.code.run_gencode_remote( cdist_object) @@ -822,20 +824,20 @@ class Config: # Execute if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Processing %s" % (cdist_object.name)) + self.log.info("Processing %s", cdist_object.name) if not self.dry_run: if cdist_object.code_local: - self.log.trace("Executing local code for %s" - % (cdist_object.name)) + self.log.trace("Executing local code for %s", + cdist_object.name) self.code.run_code_local(cdist_object) if cdist_object.code_remote: - self.log.trace("Executing remote code for %s" - % (cdist_object.name)) + self.log.trace("Executing remote code for %s", + cdist_object.name) self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) # Mark this object as done - self.log.trace("Finishing run of " + cdist_object.name) + self.log.trace("Finishing run of %s", cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 51d61e04..bb3a65bd 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error): self.message = message or 'Illegal object id' def __str__(self): - return '%s: %s' % (self.message, self.object_id) + return '{}: {}'.format(self.message, self.object_id) class MissingObjectIdError(cdist.Error): def __init__(self, type_name): self.type_name = type_name - self.message = ("Type %s requires object id (is not a " - "singleton type)") % self.type_name + self.message = ("Type {} requires object id (is not a " + "singleton type)").format(self.type_name) def __str__(self): - return '%s' % (self.message) + return '{}'.format(self.message) class CdistObject: @@ -142,7 +142,7 @@ class CdistObject: if self.object_marker in self.object_id.split(os.sep): raise IllegalObjectIdError( self.object_id, ('object_id may not contain ' - '\'%s\'') % self.object_marker) + '\'{}\'').format(self.object_marker)) if '//' in self.object_id: raise IllegalObjectIdError( self.object_id, 'object_id may not contain //') @@ -189,7 +189,7 @@ class CdistObject: object_id=object_id) def __repr__(self): - return '' % self.name + return ''.format(self.name) def __eq__(self, other): """define equality as 'name is the same'""" @@ -247,6 +247,13 @@ class CdistObject: lambda obj: os.path.join(obj.absolute_path, 'typeorder')) typeorder_dep = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep')) + # objects without parents are objects specified in init manifest + parents = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'parents')) + # objects without children are object of types that do not reuse other + # types + children = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'children')) def cleanup(self): try: @@ -270,10 +277,10 @@ class CdistObject: os.makedirs(path, exist_ok=allow_overwrite) except EnvironmentError as error: raise cdist.Error(('Error creating directories for cdist object: ' - '%s: %s') % (self, error)) + '{}: {}').format(self, error)) def requirements_unfinished(self, requirements): - """Return state whether requirements are satisfied""" + """Return unsatisfied requirements""" object_list = [] @@ -284,3 +291,14 @@ class CdistObject: object_list.append(cdist_object) return object_list + + def has_requirements_unfinished(self, requirements): + """Return whether requirements are satisfied""" + + for requirement in requirements: + cdist_object = self.object_from_name(requirement) + + if cdist_object.state != self.STATE_DONE: + return True + + return False diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index c0329c8a..b68448bc 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error): self.source_path = os.path.realpath(self.type_absolute_path) def __str__(self): - return "Invalid type '%s' at '%s' defined at '%s'" % ( + return "Invalid type '{}' at '{}' defined at '{}'".format( self.type_path, self.type_absolute_path, self.source_path) @@ -82,9 +82,9 @@ class CdistType: yield cls(base_path, name) except InvalidTypeError as e: # ignore invalid type, log warning and continue - msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % ( - e.type_path, e.type_absolute_path, e.source_path) - cls.log.warning(msg) + cls.log.warning("Ignoring invalid type '%s' at '%s' defined" + " at '%s'", e.type_path, e.type_absolute_path, + e.source_path) # remove invalid from runtime conf dir os.remove(e.type_absolute_path) @@ -109,7 +109,7 @@ class CdistType: return cls._instances[name] def __repr__(self): - return '' % self.name + return ''.format(self.name) def __eq__(self, other): return isinstance(other, self.__class__) and self.name == other.name diff --git a/cdist/core/code.py b/cdist/core/code.py index 1e9b4f80..12888ba4 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -122,8 +122,8 @@ class Code: def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type - script = os.path.join(self.local.type_path, - getattr(cdist_type, 'gencode_%s_path' % which)) + gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which)) + script = os.path.join(self.local.type_path, gencode_attr) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) @@ -167,8 +167,8 @@ class Code: def _run_code(self, cdist_object, which, env=None): which_exec = getattr(self, which) - script = os.path.join(which_exec.object_path, - getattr(cdist_object, 'code_%s_path' % which)) + code_attr = getattr(cdist_object, 'code_{}_path'.format(which)) + script = os.path.join(which_exec.object_path, code_attr) if which_exec.save_output_streams: stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index a3baa959..48414ade 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -131,18 +131,17 @@ class Explorer: self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.debug("Running global explorers in {} parallel jobs".format( - self.jobs)) - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for global " - "explorers run")) + self.log.debug("Running global explorers in %s parallel jobs", + self.jobs) + self.log.trace("Multiprocessing start method is %s", + multiprocessing.get_start_method()) + self.log.trace("Starting multiprocessing Pool for global explorers" + " run") args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.trace(("Multiprocessing run for global explorers " - "finished")) + self.log.trace("Multiprocessing run for global explorers finished") # logger is not pickable, so remove it when we pickle def __getstate__(self): @@ -161,8 +160,8 @@ class Explorer: self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path, self.jobs) - self.remote.run(["chmod", "0700", - "%s/*" % (self.remote.global_explorer_path)]) + self.remote.run(["chmod", "0700", "{}/*".format( + self.remote.global_explorer_path)]) def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" @@ -184,15 +183,14 @@ class Explorer: in the object. """ - self.log.verbose("Running type explorers for {}".format( - cdist_object.cdist_type)) + self.log.verbose("Running type explorers for %s", + cdist_object.cdist_type) if transfer_type_explorers: self.log.trace("Transferring type explorers for type: %s", cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) else: - self.log.trace(("No need for transferring type explorers for " - "type: %s"), + self.log.trace("No need for transferring type explorers for %s", cdist_object.cdist_type) self.log.trace("Transferring object parameters for object: %s", cdist_object.name) @@ -236,15 +234,15 @@ class Explorer: remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.trace(("Skipping retransfer of type explorers " - "for: %s"), cdist_type) + self.log.trace("Skipping retransfer of type explorers for: %s", + cdist_type) else: source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) self.remote.transfer(source, destination) - self.remote.run(["chmod", "0700", "%s/*" % (destination)]) + self.remote.run(["chmod", "0700", "{}/*".format(destination)]) self._type_explorers_transferred.append(cdist_type.name) def transfer_object_parameters(self, cdist_object): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 390340d4..09e74dac 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -80,13 +80,12 @@ class NoInitialManifestError(cdist.Error): if user_supplied: if os.path.islink(manifest_path): - self.message = "%s: %s -> %s" % ( - msg_header, manifest_path, - os.path.realpath(manifest_path)) + self.message = "{}: {} -> {}".format( + msg_header, manifest_path, os.path.realpath(manifest_path)) else: - self.message = "%s: %s" % (msg_header, manifest_path) + self.message = "{}: {}".format(msg_header, manifest_path) else: - self.message = "%s" % (msg_header) + self.message = "{}".format(msg_header) def __str__(self): return repr(self.message) @@ -107,7 +106,7 @@ class Manifest: self._open_logger() self.env = { - 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__global': self.local.base_path, @@ -161,7 +160,7 @@ class Manifest: raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" - self.log.verbose("Running initial manifest " + initial_manifest) + self.log.verbose("Running initial manifest %s", initial_manifest) which = "init" if self.local.save_output_streams: stderr_path = os.path.join(self.local.stderr_base_path, which) diff --git a/cdist/emulator.py b/cdist/emulator.py index a2bdc3d4..c55b47d2 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest class MissingRequiredEnvironmentVariableError(cdist.Error): def __init__(self, name): self.name = name - self.message = ("Emulator requires the environment variable %s to be " - "setup" % self.name) + self.message = ("Emulator requires the environment variable {} to be " + "setup").format(self.name) def __str__(self): return self.message @@ -106,8 +106,9 @@ class Emulator: self.save_stdin() self.record_requirements() self.record_auto_requirements() - self.log.trace("Finished %s %s" % ( - self.cdist_object.path, self.parameters)) + self.record_parent_child_relationships() + self.log.trace("Finished %s %s", self.cdist_object.path, + self.parameters) def __init_log(self): """Setup logging facility""" @@ -169,7 +170,7 @@ class Emulator: # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.trace('Args: %s' % self.args) + self.log.trace('Args: %s', self.args) def init_object(self): # Initialize object - and ensure it is not in args @@ -230,18 +231,18 @@ class Emulator: if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: obj_params = self._object_params_in_context() if obj_params != self.parameters: - errmsg = ("Object %s already exists with conflicting " - "parameters:\n%s: %s\n%s: %s" % ( + errmsg = ("Object {} already exists with conflicting " + "parameters:\n{}: {}\n{}: {}").format( self.cdist_object.name, " ".join(self.cdist_object.source), obj_params, self.object_source, - self.parameters)) + self.parameters) raise cdist.Error(errmsg) else: if self.cdist_object.exists: - self.log.debug(('Object %s override forced with ' - 'CDIST_OVERRIDE'), self.cdist_object.name) + self.log.debug('Object %s override forced with CDIST_OVERRIDE', + self.cdist_object.name) self.cdist_object.create(True) else: self.cdist_object.create() @@ -259,8 +260,8 @@ class Emulator: parent = self.cdist_object.object_from_name(__object_name) parent.typeorder.append(self.cdist_object.name) if self._order_dep_on(): - self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' - ' for %s'), depname, parent.name) + self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s', + depname, parent.name) parent.typeorder_dep.append(depname) elif self._order_dep_on(): self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', @@ -291,7 +292,7 @@ class Emulator: fd.write(chunk) chunk = self._read_stdin() except EnvironmentError as e: - raise cdist.Error('Failed to read from stdin: %s' % e) + raise cdist.Error('Failed to read from stdin: {}'.format(e)) def record_requirement(self, requirement): """record requirement and return recorded requirement""" @@ -300,16 +301,14 @@ class Emulator: try: cdist_object = self.cdist_object.object_from_name(requirement) except core.cdist_type.InvalidTypeError as e: - self.log.error(("%s requires object %s, but type %s does not" - " exist. Defined at %s" % ( - self.cdist_object.name, - requirement, e.name, self.object_source))) + self.log.error("%s requires object %s, but type %s does not" + " exist. Defined at %s", self.cdist_object.name, + requirement, e.name, self.object_source) raise except core.cdist_object.MissingObjectIdError: - self.log.error(("%s requires object %s without object id." - " Defined at %s" % (self.cdist_object.name, - requirement, - self.object_source))) + self.log.error("%s requires object %s without object id." + " Defined at %s", self.cdist_object.name, + requirement, self.object_source) raise self.log.debug("Recording requirement %s for %s", @@ -379,10 +378,9 @@ class Emulator: self.env['require'] += " " + lastcreatedtype else: self.env['require'] = lastcreatedtype - self.log.debug(("Injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s"), - lastcreatedtype, - self.cdist_object.name) + self.log.debug("Injecting require for" + " CDIST_ORDER_DEPENDENCY: %s for %s", + lastcreatedtype, self.cdist_object.name) except IndexError: # if no second last line, we are on the first type, # so do not set a requirement @@ -390,7 +388,7 @@ class Emulator: if "require" in self.env: requirements = self.env['require'] - self.log.debug("reqs = " + requirements) + self.log.debug("reqs = %s", requirements) for requirement in self._parse_require(requirements): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: @@ -420,3 +418,21 @@ class Emulator: self.log.debug("Recording autorequirement %s for %s", current_object.name, parent.name) parent.autorequire.append(current_object.name) + + def record_parent_child_relationships(self): + # __object_name is the name of the object whose type manifest is + # currently executed + __object_name = self.env.get('__object_name', None) + if __object_name: + # The object whose type manifest is currently run + parent = self.cdist_object.object_from_name(__object_name) + # The object currently being defined + current_object = self.cdist_object + if current_object.name not in parent.children: + self.log.debug("Recording child %s for %s", + current_object.name, parent.name) + parent.children.append(current_object.name) + if parent.name not in current_object.parents: + self.log.debug("Recording parent %s for %s", + parent.name, current_object.name) + current_object.parents.append(parent.name) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index e0aab190..370336ad 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -152,10 +152,10 @@ class Local: def _setup_object_marker_file(self): with open(self.object_marker_file, 'w') as fd: - fd.write("%s\n" % self.object_marker_name) + fd.write("{}\n".format(self.object_marker_name)) - self.log.trace("Object marker %s saved in %s" % ( - self.object_marker_name, self.object_marker_file)) + self.log.trace("Object marker %s saved in %s", + self.object_marker_name, self.object_marker_file) def _init_cache_dir(self, cache_dir): home_dir = cdist.home_dir() @@ -184,7 +184,7 @@ class Local: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: %s" % command) + "list or tuple argument expected, got: {}".format(command)) quiet = self.quiet_mode or quiet_mode do_save_output = save_output and not quiet and self.save_output_streams @@ -289,14 +289,12 @@ class Local: return cache_subpath def save_cache(self, start_time=time.time()): - self.log.trace("cache subpath pattern: {}".format( - self.cache_path_pattern)) + self.log.trace("cache subpath pattern: %s", self.cache_path_pattern) cache_subpath = self._cache_subpath(start_time, self.cache_path_pattern) - self.log.debug("cache subpath: {}".format(cache_subpath)) + self.log.debug("cache subpath: %s", cache_subpath) destination = os.path.join(self.cache_path, cache_subpath) - self.log.trace(("Saving cache: " + self.base_path + " to " + - destination)) + self.log.trace("Saving cache %s to %s", self.base_path, destination) if not os.path.exists(destination): shutil.move(self.base_path, destination) @@ -332,7 +330,7 @@ class Local: # Iterate over all directories and link the to the output dir for conf_dir in self.conf_dirs: - self.log.debug("Checking conf_dir %s ..." % (conf_dir)) + self.log.debug("Checking conf_dir %s ...", conf_dir) for sub_dir in CONF_SUBDIRS_LINKED: current_dir = os.path.join(conf_dir, sub_dir) @@ -350,12 +348,13 @@ class Local: if os.path.exists(dst): os.unlink(dst) - self.log.trace("Linking %s to %s ..." % (src, dst)) + self.log.trace("Linking %s to %s ...", src, dst) try: os.symlink(src, dst) except OSError as e: - raise cdist.Error("Linking %s %s to %s failed: %s" % ( - sub_dir, src, dst, e.__str__())) + raise cdist.Error( + "Linking {} {} to {} failed: {}".format( + sub_dir, src, dst, e.__str__())) def _link_types_for_emulator(self): """Link emulator to types""" @@ -368,5 +367,5 @@ class Local: os.symlink(src, dst) except OSError as e: raise cdist.Error( - "Linking emulator from %s to %s failed: %s" % ( + "Linking emulator from {} to {} failed: {}".format( src, dst, e.__str__())) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index e5af2f34..af9e70cb 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -24,12 +24,10 @@ import os import glob import subprocess import logging -import multiprocessing import cdist import cdist.exec.util as util import cdist.util.ipaddr as ipaddr -from cdist.mputil import mp_pool_run def _wrap_addr(addr): @@ -176,19 +174,19 @@ class Remote: # create archive tarpath, fcnt = autil.tar(source, self.archiving_mode) if tarpath is None: - self.log.trace(("Files count {} is lower than {} limit, " - "skipping archiving").format( - fcnt, autil.FILES_LIMIT)) + self.log.trace("Files count %d is lower than %d limit, " + "skipping archiving", + fcnt, autil.FILES_LIMIT) else: - self.log.trace(("Archiving mode, tarpath: %s, file count: " - "%s"), tarpath, fcnt) + self.log.trace("Archiving mode, tarpath: %s, file count: " + "%s", tarpath, fcnt) # get archive name tarname = os.path.basename(tarpath) self.log.trace("Archiving mode tarname: %s", tarname) # archive path at the remote desttarpath = os.path.join(destination, tarname) - self.log.trace( - "Archiving mode desttarpath: %s", desttarpath) + self.log.trace("Archiving mode desttarpath: %s", + desttarpath) # transfer archive to the remote side self.log.trace("Archiving mode: transferring") self._transfer_file(tarpath, desttarpath) @@ -262,9 +260,10 @@ class Remote: # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g. # /bin/csh will execute this script in the right way. if env: - remote_env = [" export %s=%s;" % item for item in env.items()] - string_cmd = ("/bin/sh -c '" + " ".join(remote_env) + - " ".join(command) + "'") + remote_env = [" export {env[0]}={env[1]};".format(env=item) + for item in env.items()] + string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env), + " ".join(command)) cmd.append(string_cmd) else: cmd.extend(command) @@ -278,7 +277,7 @@ class Remote: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: %s" % command) + "list or tuple argument expected, got: {}".format(command)) close_stdout = False close_stderr = False diff --git a/cdist/install.py b/cdist/install.py index 7c894fe5..d077baef 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -47,4 +47,4 @@ class Install(cdist.config.Config): yield cdist_object else: self.log.debug("Running in install mode, ignoring non install" - "object: {0}".format(cdist_object)) + "object: %s", cdist_object) diff --git a/cdist/inventory.py b/cdist/inventory.py index 0387f326..106052a2 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -92,7 +92,7 @@ class Inventory: self.init_db() def init_db(self): - self.log.trace("Init db: {}".format(self.db_basedir)) + self.log.trace("Init db: %s", self.db_basedir) if not os.path.exists(self.db_basedir): os.makedirs(self.db_basedir, exist_ok=True) elif not os.path.isdir(self.db_basedir): @@ -182,9 +182,9 @@ class Inventory: configuration = cfg.get_config(section='GLOBAL') determine_default_inventory_dir(args, configuration) - log.debug("Using inventory: {}".format(args.inventory_dir)) - log.trace("Inventory args: {}".format(vars(args))) - log.trace("Inventory command: {}".format(args.subcommand)) + log.debug("Using inventory: %s", args.inventory_dir) + log.trace("Inventory args: %s", vars(args)) + log.trace("Inventory command: %s", args.subcommand) if args.subcommand == "list": c = InventoryList(hosts=args.host, istag=args.tag, @@ -237,16 +237,16 @@ class InventoryList(Inventory): def _do_list(self, it_tags, it_hosts, check_func): if (it_tags is not None): param_tags = set(it_tags) - self.log.trace("param_tags: {}".format(param_tags)) + self.log.trace("param_tags: %s", param_tags) else: param_tags = set() for host in it_hosts: - self.log.trace("host: {}".format(host)) + self.log.trace("host: %s", host) tags = self._get_host_tags(host) if tags is None: - self.log.debug("Host \'{}\' not found, skipped".format(host)) + self.log.debug("Host \'%s\' not found, skipped", host) continue - self.log.trace("tags: {}".format(tags)) + self.log.trace("tags: %s", tags) if check_func(tags, param_tags): yield host, tags @@ -308,11 +308,11 @@ class InventoryHost(Inventory): def _action(self, host): if self.action == "add": - self.log.debug("Adding host \'{}\'".format(host)) + self.log.debug("Adding host \'%s\'", host) elif self.action == "del": - self.log.debug("Deleting host \'{}\'".format(host)) + self.log.debug("Deleting host \'%s\'", host) hostpath = self._host_path(host) - self.log.trace("hostpath: {}".format(hostpath)) + self.log.trace("hostpath: %s", hostpath) if self.action == "add" and not os.path.exists(hostpath): self._new_hostpath(hostpath) else: @@ -372,23 +372,23 @@ class InventoryTag(Inventory): print("Host \'{}\' does not exist, skipping".format(host), file=sys.stderr) return - self.log.trace("existing host_tags: {}".format(host_tags)) + self.log.trace("existing host_tags: %s", host_tags) if self.action == "del" and self.all: host_tags = set() else: for tag in self.input_tags: if self.action == "add": - self.log.debug("Adding tag \'{}\' for host \'{}\'".format( - tag, host)) + self.log.debug("Adding tag \'%s\' for host \'%s\'", + tag, host) host_tags.add(tag) elif self.action == "del": - self.log.debug("Deleting tag \'{}\' for host " - "\'{}\'".format(tag, host)) + self.log.debug("Deleting tag \'%s\' for host \'%s\'", + tag, host) if tag in host_tags: host_tags.remove(tag) - self.log.trace("new host tags: {}".format(host_tags)) + self.log.trace("new host tags: %s", host_tags) if not self._write_host_tags(host, host_tags): - self.log.trace("{} does not exist, skipped".format(host)) + self.log.trace("%s does not exist, skipped", host) def run(self): if self.allhosts: diff --git a/cdist/log.py b/cdist/log.py index 113f3b4c..62e457fe 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,25 +36,27 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') + logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(msg, *args, **kwargs): - logging.log(logging.VERBOSE, msg, *args, **kwargs) +def _verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, args, **kwargs) -logging.verbose = _verbose +logging.Logger.verbose = _verbose + logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(msg, *args, **kwargs): - logging.log(logging.TRACE, msg, *args, **kwargs) +def _trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) -logging.trace = _trace +logging.Logger.trace = _trace class CdistFormatter(logging.Formatter): diff --git a/cdist/message.py b/cdist/message.py index ffa8c2bb..0c4e21a6 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -70,7 +70,7 @@ class Message: with open(self.global_messages, 'a') as fd: for line in content: - fd.write("%s:%s" % (self.prefix, line)) + fd.write("{}:{}".format(self.prefix, line)) def merge_messages(self): self._merge_messages() diff --git a/cdist/preos.py b/cdist/preos.py index f8a5dd67..45711a41 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir): c = cm[1] yield from preos_plugin(c) except ImportError as e: - log.warning("Cannot import '{}': {}".format(module_name, e)) + log.warning("Cannot import '%s': %s", module_name, e) def find_preos_plugins(): @@ -102,7 +102,7 @@ class PreOS: parser.add_argument('remainder_args', nargs=argparse.REMAINDER) args = parser.parse_args(argv[1:]) cdist.argparse.handle_loglevel(args) - log.debug("preos args : {}".format(args)) + log.debug("preos args : %s", args) conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) @@ -122,7 +122,7 @@ class PreOS: func_args = [preos, args.remainder_args, ] else: func_args = [args.remainder_args, ] - log.info("Running preos : {}".format(preos_name)) + log.info("Running preos : %s", preos_name) func(*func_args) else: raise cdist.Error( diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index a20cdb9c..f1f750ee 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -166,7 +166,7 @@ class Debian: args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir) cdist.argparse.handle_loglevel(args) - log.debug("preos: {}, args: {}".format(cls._preos_name, args)) + log.debug("preos: %s, args: %s", cls._preos_name, args) try: env = vars(args) new_env = {} @@ -190,27 +190,30 @@ class Debian: env = new_env env.update(os.environ) cls.update_env(env) - log.debug("preos: {} env: {}".format(cls._preos_name, env)) + log.debug("preos: %s env: %s", cls._preos_name, env) + + if log.getEffectiveLevel() <= logging.INFO: + info_msg = ["Running preos: {}, suite: {}, arch: {}".format( + cls._preos_name, args.suite, args.arch), ] + if args.mirror: + info_msg.append("mirror: {}".format(args.mirror)) + if args.script: + info_msg.append("script: {}".format(args.script)) + if args.bootstrap: + info_msg.append("bootstrapping") + if args.configure: + info_msg.append("configuring") + if args.pxe_boot_dir: + info_msg.append("creating PXE") + if args.drive: + info_msg.append("creating bootable drive") + log.info(info_msg) + cmd = os.path.join(cls._files_dir, "code") - info_msg = ["Running preos: {}, suite: {}, arch: {}".format( - cls._preos_name, args.suite, args.arch), ] - if args.mirror: - info_msg.append("mirror: {}".format(args.mirror)) - if args.script: - info_msg.append("script: {}".format(args.script)) - if args.bootstrap: - info_msg.append("bootstrapping") - if args.configure: - info_msg.append("configuring") - if args.pxe_boot_dir: - info_msg.append("creating PXE") - if args.drive: - info_msg.append("creating bootable drive") - log.info(info_msg) - log.debug("cmd={}".format(cmd)) + log.debug("cmd=%s", cmd) subprocess.check_call(cmd, env=env, shell=True) except subprocess.CalledProcessError as e: - log.error("preos {} failed: {}".format(cls._preos_name, e)) + log.error("preos %s failed: %s", cls._preos_name, e) class Ubuntu(Debian): diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index eca4cf13..ddbe4933 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,36 +20,98 @@ # import logging +import sys +from datetime import datetime log = logging.getLogger("scan") -# define this outside of the class to not handle scapy import errors by default -def commandline(args): - log.debug(args) - - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - print('cdist scan requires scapy to be installed') - +def run(scan, args): + # We run each component in a separate process since they + # must not block on each other. processes = [] - if not args.mode: - # By default scan and trigger, but do not call any action - args.mode = ['scan', 'trigger', ] - if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interfaces) + t = scan.Trigger(interfaces=args.interface, + sleeptime=args.trigger_delay) t.start() processes.append(t) log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner(interfaces=args.interfaces, args=args) + s = scan.Scanner( + autoconfigure='config' in args.mode, + interfaces=args.interface, + name_mapper=args.name_mapper) s.start() processes.append(s) log.debug("Scanner started") for process in processes: process.join() + + +def list(scan, args): + s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) + hosts = s.list() + + # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + + # 6 colons. + ipv6_max_size = 8 * 4 + 10 + date_max_size = len(datetime.now().strftime(scan.datetime_format)) + name_max_size = 25 + + print("{} | {} | {} | {}".format( + 'name'.ljust(name_max_size), + 'address'.ljust(ipv6_max_size), + 'last seen'.ljust(date_max_size), + 'last configured'.ljust(date_max_size))) + print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) + for host in hosts: + last_seen = host.last_seen() + if last_seen: + last_seen = last_seen.strftime(scan.datetime_format) + else: + last_seen = '-' + + last_configured = host.last_configured() + if last_configured is not None: + last_configured = last_configured.strftime(scan.datetime_format) + else: + last_configured = '-' + + print("{} | {} | {} | {}".format( + host.name(default='-').ljust(name_max_size), + host.address().ljust(ipv6_max_size), + last_seen.ljust(date_max_size), + last_configured.ljust(date_max_size))) + + +# CLI processing is defined outside of the main scan class to handle +# non-available optional scapy dependency (instead of crashing mid-flight). +def commandline(args): + log.debug(args) + + # Check if we have the optional scapy dependency available. + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + log.error('cdist scan requires scapy to be installed. Exiting.') + sys.exit(1) + + # Set default operation mode. + if not args.mode: + # By default scan and trigger, but do not call any action. + args.mode = ['scan', 'trigger', ] + + if 'config' in args.mode and args.name_mapper is None: + print('--name-mapper must be specified for scanner config mode.', + file=sys.stderr) + sys.exit(1) + + # Print known hosts and exit is --list is specified - do not start + # the scanner. + if args.list: + list(scan, args) + else: + run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index b1d0e9e1..4a20f511 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,38 +19,6 @@ # # -# -# Interface to be implemented: -# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated -# scan: scan / listen for icmp6 replies -# trigger: send trigger to multicast -# config: configure newly detected hosts -# install: install newly detected hosts -# -# Scanner logic -# - save results to configdir: -# basedir = ~/.cdist/scan/ -# last_seen = ~/.cdist/scan//last_seen -- record unix time -# or similar -# last_configured = ~/.cdist/scan//last_configured -- record -# unix time or similar -# last_installed = ~/.cdist/scan//last_configured -- record -# unix time or similar -# -# -# -# -# cdist scan --list -# Show all known hosts including last seen flag -# -# Logic for reconfiguration: -# -# - record when configured last time -# - introduce a parameter --reconfigure-after that takes time argument -# - reconfigure if a) host alive and b) reconfigure-after time passed -# - - from multiprocessing import Process import os import logging @@ -61,7 +29,84 @@ import datetime import cdist.config +logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") +datetime_format = '%Y-%m-%d %H:%M:%S' + + +class Host(object): + def __init__(self, addr, outdir, name_mapper=None): + self.addr = addr + self.workdir = os.path.join(outdir, addr) + self.name_mapper = name_mapper + + os.makedirs(self.workdir, exist_ok=True) + + def __get(self, key, default=None): + fname = os.path.join(self.workdir, key) + value = default + if os.path.isfile(fname): + with open(fname, "r") as fd: + value = fd.readline() + return value + + def __set(self, key, value): + fname = os.path.join(self.workdir, key) + with open(fname, "w") as fd: + fd.write(f"{value}") + + def name(self, default=None): + if self.name_mapper is None: + return default + + fpath = os.path.join(os.getcwd(), self.name_mapper) + if os.path.isfile(fpath) and os.access(fpath, os.X_OK): + out = subprocess.run([fpath, self.addr], capture_output=True) + if out.returncode != 0: + return default + else: + value = out.stdout.decode() + return (default if len(value) == 0 else value) + else: + return default + + def address(self): + return self.addr + + def last_seen(self, default=None): + raw = self.__get('last_seen') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def last_configured(self, default=None): + raw = self.__get('last_configured') + if raw: + return datetime.datetime.strptime(raw, datetime_format) + else: + return default + + def seen(self): + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_seen', now) + + # XXX: There's no easy way to use the config module without feeding it with + # CLI args. Might as well call everything from scratch! + def configure(self): + target = self.name() or self.address() + cmd = ['cdist', 'config', '-v', target] + + fname = os.path.join(self.workdir, 'last_configuration_log') + with open(fname, "w") as fd: + log.debug("Executing: %s", cmd) + completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) + if completed_process.returncode != 0: + log.error("%s return with non-zero code %i - see %s for \ + details.", cmd, completed_process.returncode, fname) + + now = datetime.datetime.now().strftime(datetime_format) + self.__set('last_configured', now) class Trigger(object): @@ -69,12 +114,14 @@ class Trigger(object): Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces=None, verbose=False): + def __init__(self, interfaces, sleeptime, verbose=False): self.interfaces = interfaces + + # Used by scapy / send in trigger/2. self.verbose = verbose - # Wait 5 seconds before triggering again - FIXME: add parameter - self.sleeptime = 5 + # Delay in seconds between sent ICMPv6EchoRequests. + self.sleeptime = sleeptime def start(self): self.processes = [] @@ -93,9 +140,14 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest() - log.debug(f"Sending request on {interface}") - send(packet, verbose=self.verbose) + try: + log.debug("Sending ICMPv6EchoRequest on %s", interface) + packet = IPv6( + dst="ff02::1%{}".format(interface) + ) / ICMPv6EchoRequest() + send(packet, verbose=self.verbose) + except Exception as e: + log.error("Could not send ICMPv6EchoRequest: %s", e) class Scanner(object): @@ -103,41 +155,62 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces=None, args=None, outdir=None): + def __init__(self, interfaces, autoconfigure=False, outdir=None, + name_mapper=None): self.interfaces = interfaces + self.autoconfigure = autoconfigure + self.name_mapper = name_mapper + self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') + os.makedirs(self.outdir, exist_ok=True) + + self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = pkg['IPv6'].src - log.verbose(f"Host {host} is alive") + host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) + if host.name(): + log.verbose("Host %s (%s) is alive", host.name(), + host.address()) + else: + log.verbose("Host %s is alive", host.address()) - dir = os.path.join(self.outdir, host) - fname = os.path.join(dir, "last_seen") + host.seen() - now = datetime.datetime.now() + # Configure if needed. + if self.autoconfigure and \ + host.last_configured(default=datetime.datetime.min) + \ + self.config_delay < datetime.datetime.now(): + self.config(host) - os.makedirs(dir, exist_ok=True) + def list(self): + hosts = [] + for addr in os.listdir(self.outdir): + hosts.append(Host(addr, self.outdir, self.name_mapper)) - # FIXME: maybe adjust the format so we can easily parse again - with open(fname, "w") as fd: - fd.write(f"{now}\n") + return hosts - def config(self): - """ - Configure a host + def config(self, host): + if host.name() is None: + log.debug("config - could not resolve name for %s, aborting.", + host.address()) + return - - Assume we are only called if necessary - - However we need to ensure to not run in parallel - - Maybe keep dict storing per host processes - - Save the result - - Save the output -> probably aligned to config mode + previous_config_process = self.running_configs.get(host.name()) + if previous_config_process is not None and \ + previous_config_process.is_alive(): + log.debug("config - is already running for %s, aborting.", + host.name()) - """ + log.info("config - running against host %s (%s).", host.name(), + host.address()) + p = Process(target=host.configure()) + p.start() + self.running_configs[host.name()] = p def start(self): self.process = Process(target=self.scan) @@ -148,47 +221,9 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - - -if __name__ == '__main__': - t = Trigger(interfaces=["wlan0"]) - t.start() - - # Scanner can listen on many interfaces at the same time - s = Scanner(interfaces=["wlan0"]) - s.scan() - - # Join back the trigger processes - t.join() - - # Test in my lan shows: - # [18:48] bridge:cdist% ls -1d fe80::* - # fe80::142d:f0a5:725b:1103 - # fe80::20d:b9ff:fe49:ac11 - # fe80::20d:b9ff:fe4c:547d - # fe80::219:d2ff:feb2:2e12 - # fe80::21b:fcff:feee:f446 - # fe80::21b:fcff:feee:f45c - # fe80::21b:fcff:feee:f4b1 - # fe80::21b:fcff:feee:f4ba - # fe80::21b:fcff:feee:f4bc - # fe80::21b:fcff:feee:f4c1 - # fe80::21d:72ff:fe86:46b - # fe80::42b0:34ff:fe6f:f6f0 - # fe80::42b0:34ff:fe6f:f863 - # fe80::42b0:34ff:fe6f:f9b2 - # fe80::4a5d:60ff:fea1:e55f - # fe80::77a3:5e3f:82cc:f2e5 - # fe80::9e93:4eff:fe6c:c1f4 - # fe80::ba69:f4ff:fec5:6041 - # fe80::ba69:f4ff:fec5:8db7 - # fe80::bad8:12ff:fe65:313d - # fe80::bad8:12ff:fe65:d9b1 - # fe80::ce2d:e0ff:fed4:2611 - # fe80::ce32:e5ff:fe79:7ea7 - # fe80::d66d:6dff:fe33:e00 - # fe80::e2ff:f7ff:fe00:20e6 - # fe80::f29f:c2ff:fe7c:275e + try: + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + except Exception as e: + log.error("Could not start listener: %s", e) diff --git a/cdist/shell.py b/cdist/shell.py index 04a68937..05803556 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -65,7 +65,7 @@ class Shell: def _init_environment(self): self.env = os.environ.copy() additional_env = { - 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), + 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__cdist_manifest': "cdist shell", diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index a9c20cd3..11619e96 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -86,8 +86,7 @@ class ObjectClassTestCase(test.CdistTestCase): def test_create_singleton(self): """Check whether creating an object without id (singleton) works""" - singleton = self.expected_objects[0].object_from_name( - "__test_singleton") + self.expected_objects[0].object_from_name("__test_singleton") # came here - everything fine def test_create_singleton_not_singleton_type(self): @@ -126,16 +125,16 @@ class ObjectIdTestCase(test.CdistTestCase): def test_object_id_contains_object_marker(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = ( - 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME) + illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format( + OBJECT_MARKER_NAME) with self.assertRaises(core.IllegalObjectIdError): core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker_string(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = ( - 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME) + illegal_object_id = 'object_id/may/contain_{}_in_filename'.format( + OBJECT_MARKER_NAME) core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) # if we get here, the test passed @@ -195,28 +194,32 @@ class ObjectTestCase(test.CdistTestCase): def test_path(self): self.assertEqual(self.cdist_object.path, - "__third/moon/%s" % OBJECT_MARKER_NAME) + "__third/moon/{}".format(OBJECT_MARKER_NAME)) def test_absolute_path(self): self.assertEqual(self.cdist_object.absolute_path, os.path.join(self.object_base_path, - "__third/moon/%s" % OBJECT_MARKER_NAME)) + "__third/moon/{}".format( + OBJECT_MARKER_NAME))) def test_code_local_path(self): self.assertEqual(self.cdist_object.code_local_path, - "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) + "__third/moon/{}/code-local".format( + OBJECT_MARKER_NAME)) def test_code_remote_path(self): self.assertEqual(self.cdist_object.code_remote_path, - "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) + "__third/moon/{}/code-remote".format( + OBJECT_MARKER_NAME)) def test_parameter_path(self): self.assertEqual(self.cdist_object.parameter_path, - "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) + "__third/moon/{}/parameter".format( + OBJECT_MARKER_NAME)) def test_explorer_path(self): self.assertEqual(self.cdist_object.explorer_path, - "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) + "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME)) def test_parameters(self): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index befd7b57..4b2bc3ba 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase): def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] - self.env['require'] = ( - "__file/bad/id/with/%s/inside") % self.local.object_marker_name + self.env['require'] = "__file/bad/id/with/{}/inside".format( + self.local.object_marker_name) emu = emulator.Emulator(argv, env=self.env) self.assertRaises(core.IllegalObjectIdError, emu.run) diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index a7fe384d..b23f8447 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase): args = (self.target_host,) kwargs.setdefault('base_path', self.base_path) user = getpass.getuser() - kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user) - kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user) + kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user)) + kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user)) if 'stdout_base_path' not in kwargs: stdout_path = os.path.join(self.temp_dir, 'stdout') os.makedirs(stdout_path, exist_ok=True) @@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase): r = self.create_remote(remote_exec=remote_exec, remote_copy=remote_copy) self.assertEqual(r.run('true', return_output=True), - "%s\n" % self.target_host[0]) + "{}\n".format(self.target_host[0])) def test_run_script_target_host_in_env(self): handle, remote_exec_path = self.mkstemp(dir=self.temp_dir) @@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase): with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "true"]) self.assertEqual(r.run_script(script, return_output=True), - "%s\n" % self.target_host[0]) + "{}\n".format(self.target_host[0])) def test_run_script_with_env_target_host_in_env(self): handle, script = self.mkstemp(dir=self.temp_dir) diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py index 61cd5d97..55040fc9 100644 --- a/cdist/test/message/__init__.py +++ b/cdist/test/message/__init__.py @@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase): def test_message_merge_prefix(self): """Ensure messages are merged and are prefixed""" - expectedcontent = "%s:%s" % (self.prefix, self.content) + expectedcontent = "{}:{}".format(self.prefix, self.content) out = self.message.env['__messages_out'] diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 1d76fd76..09e9cc19 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -30,7 +30,7 @@ class AbsolutePathRequiredError(cdist.Error): self.path = path def __str__(self): - return 'Absolute path required, got: %s' % self.path + return 'Absolute path required, got: {}'.format(self.path) class FileList(collections.MutableSequence): @@ -218,7 +218,7 @@ class FileBasedProperty: def _get_attribute(self, instance, owner): name = self._get_property_name(owner) - attribute_name = '__%s' % name + attribute_name = '__{}'.format(name) if not hasattr(instance, attribute_name): path = self._get_path(instance) attribute_instance = self.attribute_class(path) diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 95ca74ee..d9e5f498 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -42,8 +42,7 @@ def resolve_target_host_name(host, family=0): # gethostbyaddr returns triple # (hostname, aliaslist, ipaddrlist) host_name = socket.gethostbyaddr(ip_addr)[0] - log.debug("derived host_name for host \"{}\": {}".format( - host, host_name)) + log.debug("derived host_name for host \"%s\": %s", host, host_name) except (socket.gaierror, socket.herror) as e: # in case of error provide empty value host_name = '' @@ -54,8 +53,7 @@ def resolve_target_fqdn(host): log = logging.getLogger(host) try: host_fqdn = socket.getfqdn(host) - log.debug("derived host_fqdn for host \"{}\": {}".format( - host, host_fqdn)) + log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn) except socket.herror as e: # in case of error provide empty value host_fqdn = '' diff --git a/docs/changelog b/docs/changelog index 88dda0aa..693d028f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,46 @@ Changelog --------- next: + * Explorer machine_type: Rewrite (Dennis Camera) + +6.9.8: 2021-08-24 + * Type __rsync: Rewrite (Ander Punnar) + * New type: __apt_pin (Daniel Fancsali) + * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) + * Core: Fix logging bug (Dennis Camera) + * Build: Improve Makefile compatibility (Evilham) + * Type __filesystem: Support ubuntu (Joachim Desroches) + * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) + * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) + * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) + * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) + * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) + +6.9.7: 2021-07-10 + * New type: __postgres_conf (Beni Ruef, Dennis Camera) + * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) + * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) + * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) + * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) + * Type __git: Fix group explorer (Ander Punnar) + * Type __pyvenv: Fix group explorer (Dennis Camera) + * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) + * Type __debconf_set_selections: Add state explorer (Dennis Camera) + * Core: Implement usable cdist scan (Timothée Floure) + * New type: __snakeoil_cert (Ander Punnar) + * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) + +6.9.6: 2021-04-20 + * Type __pyvenv: Fix user example in man page (Dennis Camera) + * Core: config: Make local state directory available to custom remotes (Steven Armstrong + * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) + * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) + * Core: Maintain object relationship graph in cdist cache (Darko Poljak) + * Type __git: Fix numeric owner and group handling (Dennis Camera) + * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) + * Type __download: Make sum parameter optional (Ander Punnar) + +6.9.5: 2021-02-28 * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) * Explorer memory: Fix result units; support Solaris (Dennis Camera) @@ -118,7 +158,7 @@ next: * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -183,7 +223,7 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1042,9 +1082,9 @@ next: * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst index d2d2d56c..d4159e77 100644 --- a/docs/src/cdist-cache.rst +++ b/docs/src/cdist-cache.rst @@ -61,6 +61,14 @@ Object cache overview ~~~~~~~~~~~~~~~~~~~~~ Each object under :strong:`object` directory has its own structure. +autorequire + file containing a list of object auto requirements + +children + file containing a list of object children, i.e. objects of types that this + type reuses (along with 'parents' it is used for maintaining parent-child + relationship graph) + code-local code generated from gencode-local, present only if something is generated @@ -80,6 +88,15 @@ parameter directory containing type parameter named files containing parameter values +parents + file containing a list of object parents, i.e. objects of types that reuse + this type (along with 'children' it is used for maintaining parent-child + relationship graph); objects without parents are objects specified in init + manifest + +require + file containing a list of object requirements + source this type's source (init manifest) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 18863145..390ab9ec 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx (for building html docs and/or the man pages) + * sphinx with the rtd theme (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst new file mode 100644 index 00000000..86b7fab6 --- /dev/null +++ b/docs/src/cdist-scan.rst @@ -0,0 +1,84 @@ +Scan +===== + +Description +----------- +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. It is especially useful in netboot-based environment where hosts +boot unconfigured, and to ensure your infrastructure stays in sync with your +configuration. + +This feature is still consider to be in **beta** stage, and only operate on +IPv6 (including link-local). + +Usage (Examples) +---------------- + +Discover hosts on local network and configure those whose name is resolved by +the name mapper script. + +.. code-block:: sh + + $ cdist scan --beta --interface eth0 \ + --mode scan --name-mapper path/to/script \ + --mode trigger --mode config + +List known hosts and exit. + +.. code-block:: sh + + $ cdist scan --beta --list --name-mapper path/to/script + +Please refer to `cdist(1)` for a detailed list of parameters. + +Modes +----- + +The scanner has 3 modes that can be independently toggled. If the `--mode` +parameter is not specified, only `tigger` and `scan` are enabled (= hosts are +not configured). + +trigger + Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to + trigger detection by the `scan` module. + +scan + Watch for incoming ICMPv6 replies and optionally configure detected hosts. + +config + Enable configuration of hosts detected by `scan`. + +Name Mapper Script +------------------ + +The name mapper script takes an IPv6 address as first argument and writes the +resolved name to stdout - if any. The script must be executable. + +Simplest script: + +.. code-block:: sh + + #!/bin/sh + + case "$1" in + "fe80::20d:b9ff:fe57:3524") + printf "my-host-01" + ;; + "fe80::7603:bdff:fe05:89bb") + printf "my-host-02" + ;; + esac + +Resolving name from `PTR` DNS record: + +.. code-block:: sh + + #!/bin/sh + + for cmd in dig sed; do + if ! command -v $cmd > /dev/null; then + exit 1 + fi + done + + dig +short -x "$1" | sed -e 's/.$//' diff --git a/docs/src/conf.py b/docs/src/conf.py index 47765413..a3dfafca 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2020' +copyright = 'ungleich GmbH 2021' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..369d5309 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,6 +34,7 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos + cdist-scan cdist-integration cdist-reference cdist-best-practice diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 0ecb4a61..599ec3b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,6 +88,9 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] + [-d CONFIG_DELAY] [-t TRIGGER_DELAY] + DESCRIPTION ----------- @@ -641,6 +644,31 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +SCAN +---- + +Runs cdist as a daemon that discover/watch on hosts and reconfigure them +periodically. + +**-I INTERFACE, --interfaces INTERFACE** + Interface to listen on. Can be specified multiple times. + +**-m MODE, --mode MODE** + Scanner components to enable. Can be specified multiple time to enable more + than one component. Supported modes are: scan, trigger and config. Defaults + to tiggger and scan. + +**--name-mapper PATH_TO_SCRIPT** + Path to script used to resolve a remote host name from an IPv6 address. + +**--list** + List known hosts and exit. + +**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** + How long (seconds) to wait before reconfiguring after last try (config mode only). + +**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** + How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION -------------