diff --git a/Makefile b/Makefile index 89286310..3712511c 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man -SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html -SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean +SPHINXM=make -C $(DOCS_SRC_DIR) man +SPHINXH=make -C $(DOCS_SRC_DIR) html +SPHINXC=make -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/README.md b/README.md index a468dd86..de6901c7 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ [libera](https://libera.chat) +IRC: ``#cdist`` @ freenode Matrix: ``#cdist:ungleich.ch`` -Matrix and IRC are bridged. +Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist diff --git a/cdist/argparse.py b/cdist/argparse.py index f17315e7..cadac39a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,31 +485,19 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger', 'config']) - parser['scan'].add_argument( - '--list', - action='store_true', - help='List the known hosts and exit') + choices=['scan', 'trigger']) parser['scan'].add_argument( '--config', action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interface', - action='append', default=[], required=True, + '-I', '--interfaces', + action='append', default=[], help='On which interfaces to scan/trigger') parser['scan'].add_argument( - '--name-mapper', - action='store', default=None, - help='Map addresses to names, required for config mode') - parser['scan'].add_argument( - '-d', '--config-delay', - action='store', default=3600, type=int, - help='How long (seconds) to wait before reconfiguring after last try') - parser['scan'].add_argument( - '-t', '--trigger-delay', - action='store', default=5, type=int, - help='How long (seconds) to wait between ICMPv6 echo requests') + '-d', '--delay', + action='store', default=3600, + help='How long to wait before reconfiguring after last try') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 00646c75..1c84f4d7 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,6 +1,8 @@ -#!/bin/sh -e +#!/bin/sh # -# 2021 Dennis Camera (cdist at dtnr.ch) +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# 2020 Evilham (contact at evilham.com) # # This file is part of cdist. # @@ -17,1019 +19,91 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer tries to determine what type of machine the target to be -# configured is (container, virtual machine, bare-metal). -# -# It will print one line for each layer it can detect. -# The format of all lines is: TYPE[ VERB VENDOR] -# -# VERB does not have a special meaning, it is just for better readability. -# -# e.g. -# container -# container on lxc -# virtual by kvm-spapr -# -# The third word of each line (except the first) can be composed of different -# parts concatenated with a `-' (minus) character, with each component being -# a specification of the previous, e.g.: -# - lxc-libvirt (LXC container, managed by libvirt) -# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) -# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) -# -# If this explorer cannot collect enough information about virtualization it -# will fall back to 'physical'. -# -# Add /sbin and /usr/sbin to the path so we can find system -# binaries like dmidecode. -PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' -PATH="/sbin:/usr/sbin:${PATH}" -export PATH +os=$("$__explorer/os") -arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') -uname_s=$(uname -s) - - -is_command() { command -v "$1" >/dev/null 2>&1; } - -files_same() { - # shellcheck disable=SC2012 - LC_ALL=C df -P "$1" "$2" 2>/dev/null | { - read -r _ # skip header line - read -r fs1 _ _ _ _ mp1 - read -r fs2 _ _ _ _ mp2 - test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 - } && - ls -1Ldi "$1" "$2" 2>/dev/null | { - read -r ino1 _ - read -r ino2 _ - test "${ino1}" = "${ino2}" || return 1 - } +vendor_string_to_machine_type() { + for vendor in vmware bochs kvm qemu virtualbox bhyve; do + if echo "${1}" | grep -q -i "${vendor}"; then + if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then + vendor="kvm" + fi + echo "virtual_by_${vendor}" + exit + fi + done } -is_oneof() ( - x=$1; shift - for y - do - test "${x}" = "${y}" || continue - return 0 - done - return 1 -) - -tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } - -# shellcheck disable=SC2086 -glob_exists() { set -- $1; test -e "$1"; } - -get_dmi_field() { - if is_oneof "${uname_s}" NetBSD - then - case $1 - in - (system-manufacturer) _mib=machdep.dmi.system-vendor ;; - (system-product-name) _mib=machdep.dmi.system-product ;; - (system-version|system-uuid) _mib=machdep.dmi.$1 ;; - (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; - (biod-release-date) _mib=machdep.dmi.bios-date ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - if is_command dmidecode - then - dmidecode -s "$1" - elif test -d "${dmi_sysfs-}" - then - case $1 - in - (system-manufacturer) _filename=sys_vendor ;; - (system-product-name) _filename=product_name ;; - (*) _filename=$(echo "$1" | tr - _) ;; - esac - if test -r "${dmi_sysfs-}/${_filename}" - then - cat "${dmi_sysfs}/${_filename}" - fi - unset _filename - elif test "${uname_s}" = OpenBSD - then - # NOTE: something similar to system-manufacutrer and system-product-name - # is available on OpenBSD in sysctl - case $1 - in - (system-manufacturer) _mib=hw.vendor ;; - (system-product-name) _mib=hw.product ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - return 1 -} - -has_cpuinfo() { test -e /proc/cpuinfo; } - -get_sysctl() { - is_command sysctl && sysctl -n "$1" 2>/dev/null -} - -detected_layer() { - test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" -} - - -# Check for chroot - -has_chroot_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' -} - -check_chroot_systemd() { - systemd-detect-virt -r -} - -has_chroot_debian_ischroot() { - is_command ischroot -} - -check_chroot_debian_ischroot() { - ischroot --default-false -} - -has_chroot_procfs() { - test -d /proc/ -} - -check_chroot_procfs() ( - is_chroot=false # default - if test -e /proc/1/root && ! files_same /proc/1/root / - then - is_chroot=true - fi - if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo - then - has_mountinfo=true - cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true - fi - - if ${is_chroot} - then - # try to determine where the chroot has been mounted - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # escape chroot and find mount point by inode - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - elif ${has_mountinfo} - then - while read -r mntid _ _ _ cmntpnt _ - do - read -r _ _ _ _ hmntpnt _ <<-EOF - $(grep -e "^$((mntid)) " /proc/1/mountinfo) - EOF - printf '%s\n' "${hmntpnt%${cmntpnt}}" - done /dev/null) && - case ${_ctengine} - in - (''|'none') - return 1 ;; - ('container-other') - return 0 ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - echo "${_ctengine}" ;; - esac -) - -has_ct_pid_1() { - test -r /run/systemd/container -o -r /proc/1/environ -} - -translate_container_name() { - case $1 - in - ('lxc') - echo lxc ;; - ('lxc-libvirt') - echo lxc-libvirt ;; - ('podman') - echo podman ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - return 1 ;; - esac - return 0 -} - -check_ct_pid_1() { - if test -r /run/systemd/container - then - translate_container_name "$(head -n1 /run/systemd/container)" \ - && return 0 - fi - - if test -r /proc/1/environ - then - translate_container_name "$( - LC_ALL=C tr '\000' '\n' /dev/null - then - # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 - echo wsl - elif test -d /var/.cagefs - then - # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs - # CageFS is not "really" a container, but it isn't a chroot either. - echo cagefs - elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status - then - # Linux-VServer - if grep -q -x -F 'VxID: 0' /proc/self/status - then - # host - return 1 - else - # guest - echo linux_vserver - fi - else - return 1 - fi -} - -check_ct_os_specific() ( - if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 - then - # FreeBSD jail - echo jail - return 0 - fi - - if is_command zonename && test "$(zonename)" != global - then - # Solaris zone - echo zone - return 0 - fi - - return 1 -) - - -# Check for hypervisor - -guess_hypervisor_from_cpu_model() { - case $1 - in - (*\ KVM\ *) - echo kvm ;; - (*\ QEMU\ *|QEMU\ *) - echo qemu ;; - (*) - return 1 ;; - esac -} - -has_vm_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' -} - -check_vm_systemd() ( - _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && - case ${_hypervisor} - in - (''|'none') - return 1 ;; - ('amazon') - echo aws ;; - ('bochs') - echo kvm ;; - ('microsoft') - # assumption - echo hyperv ;; - ('oracle') - echo virtualbox ;; - ('vm-other') - return 0 ;; - (*) - echo "${_hypervisor}" ;; - esac -) - -has_vm_cpuinfo() { has_cpuinfo; } - -check_vm_cpuinfo() { - if grep -q -F 'User Mode Linux' /proc/cpuinfo \ - || grep -q -F 'UML' /proc/cpuinfo - then - # User Mode Linux - echo uml - elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo - then - # IBM PowerVM Lx86 (Linux/x86 emulator) - echo powervm_lx86 - elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo - then - # IBM SystemZ (S/390) - if test -f /proc/sysinfo - then - if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo - then - echo kvm-s390 - return 0 - elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo - then - echo zvm - return 0 - elif grep -q -e '^LPAR ' /proc/sysinfo - then - echo zvm-lpar - return 0 - fi - fi - return 1 - else - if grep -q -e '^model name.*:' /proc/cpuinfo - then - sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ - | while read -r _cpu_model - do - guess_hypervisor_from_cpu_model "${_cpu_model}" - done \ - | sort \ - | uniq -c \ - | awk ' - { if ($1 > most_c) { most_c = $1; most_s = $2 } } - END { - if (most_s) print most_s - exit !most_s - }' \ - && return 0 - fi - return 1 - fi -} - -check_vm_arch_specific() { - case ${arch} - in - (ppc64|ppc64le) - # Check PPC64 LPAR, KVM - - # example /proc/cpuinfo line indicating 'not baremetal' - # platform : pSeries - # - # example /proc/ppc64/lparcfg systemtype line - # system_type=IBM pSeries (emulated by qemu) - - if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo - then - if test -e /proc/ppc64/lparcfg - then - # Assume LPAR, now detect shared or dedicated - if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg - then - echo powervm-shared - return 0 - else - echo powervm-dedicated - return 0 - fi - fi - fi - ;; - (sparc*) - # Check for SPARC LDoms - - if test -e /dev/mdesc - then - if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp - then - # control LDom - return 1 - else - # guest LDom - echo ldom-sparc - fi - - # MDPROP=/usr/lib/ldoms/mdprop.py - # if test -x "${MDPROP}" - # then - # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" - # then - # echo ldoms-root - # echo ldoms-io - # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" - # then - # echo ldoms-io - # fi - # fi - return 0 - fi - ;; - (i?86|x86*|amd64|i86pc) - # VMM CPUID flag denotes that this system is running under a VMM - if is_oneof "${uname_s}" Darwin - then - get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ - && return 0 - fi - if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo - then - return 0 - fi - ;; - (ia64) - if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend - then - # PV-on-HVM drivers installed in a Xen guest - echo xen-hvm - return 0 - fi - ;; - esac - return 1 -} - -has_vm_dmi() { - # Check for various products in SMBIOS/DMI. - # Note that DMI doesn't exist on all architectures (only x86 and some ARM). - # On other architectures the $dmi variable will be empty. - - if test -d /sys/class/dmi/id/ - then - dmi_sysfs=/sys/class/dmi/id - elif test -d /sys/devices/virtual/dmi/id/ - then - dmi_sysfs=/sys/devices/virtual/dmi/id - fi - - # shellcheck disable=SC2015 - { - is_command dmidecode \ - && ( - # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line - dmi_out=$(dmidecode 2>&1) \ - && ! printf '%s\n' "${dmi_out}" \ - | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' - ) \ - || test -d "${dmi_sysfs}" - } -} - -check_vm_dmi() { - case $(get_dmi_field system-product-name) - in - (*.metal) - if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' - then - # AWS EC2 bare metal -> no virtualisation - return 1 - fi - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('Google Compute Engine') - echo gce - return 0 - ;; - ('RHEV Hypervisor') - # Red Hat Enterprise Virtualization - echo rhev - return 0 - ;; - ('KVM'|'Bochs'|'KVM Virtual Machine') - echo kvm - return 0 - ;; - ('Parallels Virtual Platform') - echo parallels - return 0 - ;; - ('VirtualBox') - echo virtualbox - return 0 - ;; - ('VMware Virtual Platform') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field system-manufacturer) - in - ('Alibaba'*) - case $(get_dmi_field system-product-name) - in - ('Alibaba Cloud ECS') - echo alibaba-ecs - ;; - (*) - echo alibaba - ;; - esac - return 0 - ;; - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('innotek GmbH'|'Oracle Corporation') - echo virtualbox - return 0 - ;; - ('Joyent') - if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' - then - # SmartOS KVM - echo kvm-smartdc_hvm - return 0 - fi - ;; - ('Microsoft Corporation'*) - if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' - then - if test -e /proc/irq/7/hyperv \ - || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null - then - echo hyperv - return 0 - fi - - case $(get_dmi_field system-version) - in - (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) - echo virtualpc - return 0 - ;; - (*) - echo hyperv - return 0 - ;; - esac - fi - ;; - ('Nutanix') - # Nutanix AHV. Similar to KVM. - if test "$(get_dmi_field system-product-name)" = 'AHV' - then - echo nutanix_ahv - return 0 - fi - ;; - ('oVirt') - echo ovirt - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('QEMU') - echo qemu - return 0 - ;; - ('VMware, Inc.') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field bios-vendor) - in - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('innotek GmbH') - echo virtualbox - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('Xen') - if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' - then - # AWS on Xen - echo aws-xen - return 0 - fi - ;; - esac - - return 1 -} - -check_vm_hyp_specific() { - if is_command vmware-checkvm && vmware-checkvm >/dev/null - then - # vmware-checkvm is provided by VMware's open-vm-tools - echo vmware - return 0 - elif test -d /proc/xen - then - test -r /proc/xen/capabilities && - if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null - then - # Xen dom0 - return 1 - else - # Xen domU - echo xen - return 0 - fi - fi - return 1 -} - -has_vm_dt() { - # OpenFirmware/Das U-Boot device-tree - test -d /proc/device-tree -} - -check_vm_dt() { - case ${arch} - in - (arm|aarch64) - if test -r /proc/device-tree/hypervisor/compatible - then - if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible - then - echo xen - return 0 - elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible - then - # e.g. VMware ESXi on ARM - echo vmware - return 0 - fi - fi - if glob_exists /proc/device-tree/fw-cfg@*/compatible - then - # qemu,fw-cfg-mmio - sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 - return 0 - fi - if grep -q -F 'dummy-virt' /proc/device-tree/compatible - then - echo lkvm - return 0 - fi - ;; - (ppc64*) - if test -d /proc/device-tree/hypervisor \ - && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible - then - # We are running as a spapr KVM guest on ppc64 - echo kvm-spapr - return 0 - fi - if test -r /proc/device-tree/ibm,partition-name \ - && test -r /proc/device-tree/hmc-managed\? \ - && test -r /proc/device-tree/chosen/qemu,graphic-width - then - echo powervm - fi - ;; - esac - return 1 -} - -has_vm_sys_hypervisor() { - test -d /sys/hypervisor/ -} - -check_vm_sys_hypervisor() { - test -r /sys/hypervisor/type && - case $(head -n1 /sys/hypervisor/type) - in - (xen) - # Ordinary kernel with pv_ops. There does not seem to be - # enough information at present to tell whether this is dom0 - # or domU. - echo xen - return 0 - ;; - esac - return 1 -} - -check_vm_os_specific() { - _hyp_generic=false - - case ${uname_s} - in - (Darwin) - if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ - && test "${hv_vmm_present}" -ne 0 - then - _hyp_generic=true - fi - ;; - (FreeBSD) - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest=$(get_sysctl kern.vm_guest | tolower) && - case ${vm_guest} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${vm_guest} - in - (hv) echo hyperv ;; - (vbox) echo virtualbox ;; - (*) echo "${vm_guest}" ;; - esac - return 0 - ;; - esac - ;; - (NetBSD) - machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && - case ${machdep_hv} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${machdep_hv} - in - (hyper-v) echo hyperv ;; - (xenhvm*) echo xen-hvm ;; - (xenpv*) echo xen-pv ;; - (xen*) echo xen ;; - (*) echo "${machdep_hv}" ;; - esac - return 0 - ;; - esac - ;; - (OpenBSD) - if is_command hostctl && glob_exists /dev/pvbus[0-9]* - then - for _pvbus in /dev/pvbus[0-9]* - do - _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue - case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') - in - (KVM) echo kvm ;; - (Hyper-V) echo hyperv ;; - (VMware) echo vmware ;; - (Xen) echo xen ;; - (bhyve) echo bhyve ;; - (OpenBSD) echo openbsd_vmm ;; - esac - return 0 - done - fi - ;; - (SunOS) - diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') - # NOTE: Don't use -e or -F in Solaris grep - if printf '%s\n' "${diag_conf}" | grep -q -i QEMU - then - echo qemu - return 0 - elif printf '%s\n' "${diag_conf}" | grep -q -i VMware - then - echo vmware - return 0 - fi - ;; - (Linux) - if is_command dmesg - then - while read -r line - do - case ${line} - in - ('Booting paravirtualized kernel on ') - case $(expr "${line}" : '.* kernel on \(.*\)') - in - ('Xen') - echo xen-pv; return 0 ;; - ('bare hardware') - return 1 ;; - esac - ;; - ('Hypervisor detected') - case $(expr "${line}" : '.*: *\(.*\)') - in - ('ACRN') - echo acrn ;; - ('Jailhouse') - echo jailhouse ;; - ('KVM') - echo kvm ;; - ('Microsoft Hyper-V') - echo hyperv ;; - ('VMware') - echo vmware ;; - ('Xen HVM') - echo xen-hvm ;; - ('Xen PV') - echo xen-pv ;; - esac - return 0 - ;; - (lpar:*' under hypervisor') - return 0 ;; - esac - done <<-EOF - $(dmesg 2>/dev/null | awk ' - /Booting paravirtualized kernel on / - /Hypervisor detected: / - /lpar: .* under hypervisor/ - ') - EOF - fi - esac - - # Try to guess hypervisor based on CPU model (sysctl hw.model if available) - if cpu_model=$(get_sysctl hw.model) - then - guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 - fi - - if ${_hyp_generic} - then - # cannot say which hypervisor, but one was detected - return 0 - else - return 1 - fi -} - -run_stage() { - if type "has_$1_$2" >/dev/null 2>&1 - then - "has_$1_$2" - else - true - fi \ - && "check_$1_$2" -} - - -# Execute chroot stages - -for stage in \ - procfs debian_ischroot systemd -do - chrootpnt=$(run_stage chroot ${stage}) || continue - is_chrooted=true - detected_layer 'chroot' - if test -n "${chrootpnt}" - then - echo chroot at "${chrootpnt}" - break - fi -done -if ${is_chrooted:-false} && test -z "${chrootpnt}" -then - # could determine chroot, but not its mount point - echo chroot -fi - - -# Execute container stages - -for stage in \ - systemd pid_1 cgroup files os_specific -do - ctengine=$(run_stage ct ${stage}) || continue - detected_layer 'container' - is_contained=true - if test -n "${ctengine}" - then - echo container on "${ctengine}" - break - fi -done -if ${is_contained:-false} && test -z "${ctengine}" -then - # none of the stages could determine the specific container engine, but - # we are running in some container. - echo container -fi - - -# Execute virtual machine / hypervisor stages - -for stage in \ - systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific -do - hypervisor=$(run_stage vm ${stage}) || continue - detected_layer 'virtual machine' - is_virtual=true - if test -n "${hypervisor}" - then - echo virtual by "${hypervisor}" - break - fi -done -if ${is_virtual:-false} && test -z "${hypervisor}" -then - # none of the stages could determine the specific hypervisor, but - # we are virtual. - echo virtual -fi - - -# Fallback - -detected_layer physical +case "$os" in + "freebsd") + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" + if [ -n "${vm_guest}" ]; then + if [ "${vm_guest}" = "none" ]; then + echo "physical" + exit + fi + echo "virtual_by_${vm_guest}" + exit + fi + ;; + + "openbsd") + # OpenBSD can also use the sysctl's: hw.vendor or hw.product. + # Note we can be reasonably sure about a machine being virtualised + # as long as we can identify the virtualisation technology. + # But not so much about it being physical... + # Patches are welcome / reach out if you have better ideas. + for sysctl in hw.vendor hw.product; do + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" + done + ;; + + *) + # Defaulting to linux for compatibility with previous cdist behaviour + + if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then + echo openvz + exit + fi + + if [ -e "/proc/1/environ" ] && + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then + echo lxc + exit + fi + + if [ -r /proc/cpuinfo ]; then + # this should only exist on virtual guest machines, + # tested on vmware, xen, kvm, bhyve + if grep -q "hypervisor" /proc/cpuinfo; then + # this file is aviable in xen guest systems + if [ -r /sys/hypervisor/type ]; then + if grep -q -i "xen" /sys/hypervisor/type; then + echo virtual_by_xen + exit + fi + else + for vendor_file in /sys/class/dmi/id/product_name \ + /sys/class/dmi/id/sys_vendor \ + /sys/class/dmi/id/chasis_vendor; do + if [ -r ${vendor_file} ]; then + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(cat "${vendor_file}")" + fi + done + fi + echo "virtual_by_unknown" + exit + else + echo "physical" + exit + fi + fi + ;; +esac + +echo "unknown" diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index c6d113cf..63aba9c6 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -27,18 +27,19 @@ str2bytes() { awk -F' ' ' $2 == "B" || !$2 { print $1 } - $2 == "kB" { printf "%.f\n", ($1 * 1000) } - $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } - $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } - $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } - $2 == "kiB" { printf "%.f\n", ($1 * 1024) } - $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } - $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } - $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' + $2 == "kB" { print $1 * 1000 } + $2 == "MB" { print $1 * 1000 * 1000 } + $2 == "GB" { print $1 * 1000 * 1000 * 1000 } + $2 == "TB" { print $1 * 1000 * 1000 * 1000 * 1000 } + $2 == "kiB" { print $1 * 1024 } + $2 == "MiB" { print $1 * 1024 * 1024 } + $2 == "GiB" { print $1 * 1024 * 1024 * 1024 } + $2 == "TiB" { print $1 * 1024 * 1024 * 1024 * 1024 }' } bytes2kib() { - awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' + set -- "$(cat)" + test "$1" -gt 0 && echo $(($1 / 1024)) } diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index bbc9e4f0..3b02dedd 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,7 +1,6 @@ -#!/bin/sh -e +#!/bin/sh # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,22 +17,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# # All os variables are lower case # +# -rc_getvar() { - awk -F= -v varname="$2" ' - function unquote(s) { - if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) - return substr(s, 2, length(s) - 2) - else - return s - } - $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" -} - -case $("${__explorer:?}/os") -in +case "$("$__explorer/os")" in amazon) cat /etc/system-release ;; @@ -54,8 +43,6 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in - trixie) echo 12.99 ;; - bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; @@ -63,7 +50,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) echo 99.99 ;; + *) exit 1 esac ;; *) @@ -72,23 +59,7 @@ in esac ;; devuan) - devuan_version=$(cat /etc/devuan_version) - case ${devuan_version} - in - (*/ceres) - # ceres versions don't have a number, so we decode by codename: - case ${devuan_version} - in - (chimaera/ceres) echo 3.99 ;; - (beowulf/ceres) echo 2.99 ;; - (ascii/ceres) echo 1.99 ;; - (*) exit 1 - esac - ;; - (*) - echo "${devuan_version}" - ;; - esac + cat /etc/devuan_version ;; fedora) cat /etc/fedora-release @@ -97,20 +68,12 @@ in cat /etc/gentoo-release ;; macosx) - # NOTE: Legacy versions (< 10.3) do not support options - sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' + sw_vers -productVersion ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - if command -v freebsd-version >/dev/null 2>&1 - then - # get userland version - freebsd-version -u - else - # fallback to kernel release for FreeBSD < 10.0 - uname -r - fi + freebsd-version ;; *bsd|solaris) uname -r @@ -135,20 +98,7 @@ in fi ;; ubuntu) - if command -v lsb_release >/dev/null 2>&1 - then - lsb_release -sr - elif test -r /usr/lib/os-release - then - # fallback to /usr/lib/os-release if lsb_release is not present (like - # on minimized Ubuntu installations) - rc_getvar /usr/lib/os-release VERSION_ID - elif test -r /etc/lsb-release - then - # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old - # versions without /usr/lib/os-release. - rc_getvar /etc/lsb-release DISTRIB_RELEASE - fi + lsb_release -sr ;; alpine) cat /etc/alpine-release diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index 6fcd9212..bc47d8de 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,7 +28,6 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 - # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst deleted file mode 100644 index 4229c0cd..00000000 --- a/cdist/conf/type/__apt_pin/man.rst +++ /dev/null @@ -1,79 +0,0 @@ -cdist-type__apt_pin(7) -====================== - -NAME ----- -cdist-type__apt_pin - Manage apt pinning rules - - -DESCRIPTION ------------ -Adds/removes/edits rules to pin some packages to a specific distribution. Useful if using multiple debian repositories at the same time. (Useful, if one wants to use a few specific packages from backports or perhaps Debain testing... or even sid.) - - -REQUIRED PARAMETERS -------------------- -distribution - Specifies what distribution the package should be pinned to. Accepts both codenames (buster/bullseye/sid) and suite names (stable/testing/...). - - -OPTIONAL PARAMETERS -------------------- -package - Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. - -priority - The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) - -state - Will be passed to underlying `__file` type; see there for valid values and defaults. - - - -BOOLEAN PARAMETERS ------------------- -None. - - -EXAMPLES --------- - -.. code-block:: sh - - # Add the bullseye repo to buster, but do not install any packages by default, - # only if explicitely asked for (-1 means "never" for apt) - __apt_pin bullseye-default \ - --package "*" \ - --distribution bullseye \ - --priority -1 - - require="__apt_pin/bullseye-default" __apt_source bullseye \ - --uri http://deb.debian.org/debian/ \ - --distribution bullseye \ - --component main - - __apt_pin foo --package "foo foo-*" --distribution bullseye - - __foo # Assuming, this installs the `foo` package internally - - __package foo-plugin-extras # Assuming we also need some extra stuff - - -SEE ALSO --------- -:strong:`apt_preferences`\ (5) -:strong:`cdist-type__apt_source`\ (7) -:strong:`cdist-type__apt_backports`\ (7) -:strong:`cdist-type__file`\ (7) - -AUTHORS -------- -Daniel Fancsali - - -COPYING -------- -Copyright \(C) 2021 Daniel Fancsali. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest deleted file mode 100755 index e72a8fdd..00000000 --- a/cdist/conf/type/__apt_pin/manifest +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh -e -# -# 2021 Daniel Fancsali (fancsali@gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - - -name="$__object_id" - -os=$(cat "$__global/explorer/os") -state="$(cat "$__object/parameter/state")" - -if [ -f "$__object/parameter/package" ]; then - package="$(cat "$__object/parameter/package")" -else - package=$name -fi - -distribution="$(cat "$__object/parameter/distribution")" -priority="$(cat "$__object/parameter/priority")" - - -case "$os" in - debian|ubuntu|devuan) - ;; - *) - printf "This type is specific to Debian and it's derivatives" >&2 - exit 1 - ;; -esac - -case $distribution in - stable|testing|unstable|experimental) - pin="release a=$distribution" - ;; - *) - pin="release n=$distribution" - ;; -esac - - -__file "/etc/apt/preferences.d/$name" \ - --owner root --group root --mode 0644 \ - --state "$state" \ - --source - << EOF -Package: $package -Pin: $pin -Pin-Priority: $priority -EOF diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__apt_pin/nonparallel deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional deleted file mode 100644 index 52f01fd2..00000000 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -state -package diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required deleted file mode 100644 index 4b4e9741..00000000 --- a/cdist/conf/type/__apt_pin/parameter/required +++ /dev/null @@ -1,2 +0,0 @@ -distribution -priority diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 973b0f6c..1e8592c6 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,21 +22,7 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - -# run 'apt-get update' only if something changed with our sources.list file -# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" + printf 'apt-get update || apt-get update\n' fi diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 2d7f9030..70b59710 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,23 +18,9 @@ # along with cdist. If not, see . # - -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists -# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get $apt_opts update || apt-get $apt_opts update + apt-get update || apt-get update fi DONE diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state deleted file mode 100644 index f8a3f6c8..00000000 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh -e -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Determine current debconf selections' state. -# Prints one of: -# present: all selections are already set as they should. -# different: one or more of the selections have a different value. -# absent: one or more of the selections are not (currently) defined. -# - -test -x /usr/bin/perl || { - # cannot find perl (no perl ~ no debconf) - echo 'absent' - exit 0 -} - -linesfile="${__object:?}/parameter/line" -test -s "${linesfile}" || { - if test -s "${__object:?}/parameter/file" - then - echo absent - else - echo present - fi - exit 0 -} - -# assert __type_explorer is set (because it is used by the Perl script) -: "${__type_explorer:?}" - -/usr/bin/perl -- - "${linesfile}" <<'EOF' -use strict; -use warnings "all"; - -use Fcntl qw(:DEFAULT :flock); - -use Debconf::Db; -use Debconf::Question; - -# Extract @known... arrays from debconf-set-selections -# These values are required to distinguish flags and values in the given lines. -# DC: I couldn't think of a more ugly solution to the problem… -my @knownflags; -my @knowntypes; -my $debconf_set_selections = '/usr/bin/debconf-set-selections'; -if (-e $debconf_set_selections) { - my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; - eval `sed -n '$sed_known' '$debconf_set_selections'`; -} - -sub mungeline ($) { - my $line = shift; - chomp $line; - $line =~ s/\r$//; - return $line; -} - -sub fatal { printf STDERR @_; exit 1; } - -my $state = 'present'; - -sub state { - my $new = shift; - if ($state eq 'present' - or ($state eq 'different' and $new eq 'absent')) { - $state = $new; - } -} - - -# Load Debconf DB but manually lock on the state explorer script, -# because Debconf aborts immediately if executed concurrently. -# This is not really an ideal solution because the Debconf DB could be locked by -# another process (e.g. apt-get), but no way to achieve this could be found. -# If you know how to, please provide a patch. -my $lockfile = "%ENV{'__type_explorer'}/state"; -if (open my $lock_fh, '+<', $lockfile) { - flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; -} -{ - Debconf::Db->load(readonly => 'true'); -} - - -while (<>) { - # Read and process lines (taken from debconf-set-selections) - $_ = mungeline($_); - while (/\\$/ && ! eof) { - s/\\$//; - $_ .= mungeline(<>); - } - next if /^\s*$/ || /^\s*\#/; - - my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ - or fatal "invalid line: %s\n", $_; - $content = '' unless defined $content; - - - # Compare is and should state - my $q = Debconf::Question->get($label); - - unless (defined $q) { - # probably a preseed - state 'absent'; - next; - } - - if (grep { $_ eq $q->type } @knownflags) { - # This line wants to set a flag, presumably. - if ($q->flag($q->type) ne $content) { - state 'different'; - } - } else { - # Otherwise, it's probably a value… - if ($q->value ne $content) { - state 'different'; - } - - unless (grep { $_ eq $owner } (split /, /, $q->owners)) { - state 'different'; - } - } -} - -printf "%s\n", $state; -EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 9ba28f09..e99aef40 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,37 +17,16 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# +# Setup selections +# -if test -f "${__object:?}/parameter/line" -then - filename="${__object:?}/parameter/line" -elif test -s "${__object:?}/parameter/file" -then - filename=$(cat "${__object:?}/parameter/file") - if test "${filename}" = '-' - then - filename="${__object:?}/stdin" - fi -else - printf 'Neither --line nor --file set.\n' >&2 - exit 1 +filename="$(cat "$__object/parameter/file")" + +if [ "$filename" = "-" ]; then + filename="$__object/stdin" fi -# setting no lines makes no sense -test -s "${filename}" || exit 0 - -state_is=$(cat "${__object:?}/explorer/state") - -if test "${state_is}" != 'present' -then - cat <<-CODE - debconf-set-selections <<'EOF' - $(cat "${filename}") - EOF - CODE - - awk ' - { - printf "set %s %s %s %s\n", $1, $2, $3, $4 - }' "${filename}" >>"${__messages_out:?}" -fi +echo "debconf-set-selections << __file-eof" +cat "$filename" +echo "__file-eof" diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index fd0040ae..58c25b81 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,33 +8,15 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used +On Debian and alike systems debconf-set-selections(1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- -cf. ``--line``. - - -OPTIONAL PARAMETERS -------------------- file - Use the given filename as input for :strong:`debconf-set-selections`\ (1) - If filename is ``-``, read from stdin. - - **This parameter is deprecated, because it doesn't work with state detection.** -line - A line in :strong:`debconf-set-selections`\ (1) compatible format. - This parameter can be used multiple times to set multiple options. - - (This parameter is actually required, but marked optional because the - deprecated ``--file`` is still accepted.) - - -BOOLEAN PARAMETERS ------------------- -None. + Use the given filename as input for debconf-set-selections(1) + If filename is "-", read from stdin. EXAMPLES @@ -42,29 +24,30 @@ EXAMPLES .. code-block:: sh - # Setup gitolite's gituser - __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' + # Setup configuration for nslcd + __debconf_set_selections nslcd --file /path/to/file - # Setup configuration for nslcd from a file. - # NB: Multiple lines can be passed to --line, although this can be considered a hack. - __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" + # Setup configuration for nslcd from another type + __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" + + __debconf_set_selections nslcd --file - << eof + gitolite gitolite/gituser string git + eof SEE ALSO -------- -- :strong:`cdist-type__update_alternatives`\ (7) -- :strong:`debconf-set-selections`\ (1) +:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) AUTHORS ------- -| Nico Schottelius -| Dennis Camera +Nico Schottelius COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file deleted file mode 100644 index 09db545a..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file +++ /dev/null @@ -1 +0,0 @@ -'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple deleted file mode 100644 index a999a0c2..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple +++ /dev/null @@ -1 +0,0 @@ -line diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional b/cdist/conf/type/__debconf_set_selections/parameter/required similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/optional rename to cdist/conf/type/__debconf_set_selections/parameter/required diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd new file mode 100755 index 00000000..e3e35b45 --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd @@ -0,0 +1,19 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cmd="$( cat "$__object/parameter/cmd-get" )" + +elif command -v curl > /dev/null +then + cmd="curl -L -o - '%s'" + +elif command -v fetch > /dev/null +then + cmd="fetch -o - '%s'" + +else + cmd="wget -O - '%s'" +fi + +echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get deleted file mode 100755 index 9f1cd59c..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd_get +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cat "$__object/parameter/cmd-get" -elif - command -v curl > /dev/null -then - echo "curl -sSL -o - '%s'" -elif - command -v fetch > /dev/null -then - echo "fetch -o - '%s'" -else - echo "wget -O - '%s'" -fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum deleted file mode 100755 index 84df663c..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd_sum +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh -e - -if [ ! -f "$__object/parameter/sum" ] -then - exit 0 -fi - -if [ -f "$__object/parameter/cmd-sum" ] -then - cat "$__object/parameter/cmd-sum" - exit 0 -fi - -sum_should="$( cat "$__object/parameter/sum" )" - -if echo "$sum_should" | grep -Fq ':' -then - sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" -else - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_hash='cksum' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' - then - sum_hash='md5' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' - then - sum_hash='sha1' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' - then - sum_hash='sha256' - else - echo 'hash format detection failed' >&2 - exit 1 - fi -fi - -os="$( "$__explorer/os" )" - -case "$sum_hash" in - cksum) - echo "cksum %s | awk '{print \$1\" \"\$2}'" - ;; - md5) - case "$os" in - freebsd) - echo "md5 -q %s" - ;; - *) - echo "md5sum %s | awk '{print \$1}'" - ;; - esac - ;; - sha1) - case "$os" in - freebsd) - echo "sha1 -q %s" - ;; - *) - echo "sha1sum %s | awk '{print \$1}'" - ;; - esac - ;; - sha256) - case "$os" in - freebsd) - echo "sha256 -q %s" - ;; - *) - echo "sha256sum %s | awk '{print \$1}'" - ;; - esac - ;; - *) - # we arrive here only if --sum is given with unknown format prefix - echo "unknown hash format: $sum_hash" >&2 - exit 1 - ;; -esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 8c5d5ce1..68b517c5 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,11 +1,6 @@ #!/bin/sh -e -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi +dst="/$__object_id" if [ ! -f "$dst" ] then @@ -21,19 +16,57 @@ fi sum_should="$( cat "$__object/parameter/sum" )" -if echo "$sum_should" | grep -Fq ':' +if [ -f "$__object/parameter/cmd-sum" ] then - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + # shellcheck disable=SC2059 + sum_is="$( eval "$( printf \ + "$( cat "$__object/parameter/cmd-sum" )" \ + "$dst" )" )" +else + os="$( "$__explorer/os" )" + + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" + + elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' + then + case "$os" in + freebsd) + sum_is="md5:$( md5 -q "$dst" )" + ;; + *) + sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' + then + case "$os" in + freebsd) + sum_is="sha1:$( sha1 -q "$dst" )" + ;; + *) + sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' + then + case "$os" in + freebsd) + sum_is="sha256:$( sha256 -q "$dst" )" + ;; + *) + sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" + ;; + esac + fi fi -sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" - -# shellcheck disable=SC2059 -sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" - if [ -z "$sum_is" ] then - echo 'existing destination checksum failed' >&2 + echo 'no checksum from target' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index d1b0d0d5..571d2c3c 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,133 +11,34 @@ fi url="$( cat "$__object/parameter/url" )" -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi +tmp="$( mktemp )" + +dst="/$__object_id" if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + elif command -v curl > /dev/null then - cmd="curl -sSL -o - '%s'" + cmd="curl -L -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - else - echo 'local download failed, no usable utility' >&2 + echo 'no usable locally installed utility for downloading' >&2 exit 1 fi -echo "download_tmp=\"\$( mktemp )\"" - -# shellcheck disable=SC2059 -printf "$cmd > \"\$download_tmp\"\n" "$url" - -if [ -f "$__object/parameter/sum" ] -then - sum_should="$( cat "$__object/parameter/sum" )" - - if [ -f "$__object/parameter/cmd-sum" ] - then - local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" - else - if echo "$sum_should" | grep -Fq ':' - then - sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" - - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" - else - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_hash='cksum' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' - then - sum_hash='md5' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' - then - sum_hash='sha1' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' - then - sum_hash='sha256' - else - echo 'hash format detection failed' >&2 - exit 1 - fi - fi - - case "$sum_hash" in - cksum) - local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" - ;; - md5) - if command -v md5 > /dev/null - then - local_cmd_sum="md5 -q %s" - elif - command -v md5sum > /dev/null - then - local_cmd_sum="md5sum %s | awk '{print \$1}'" - fi - ;; - sha1) - if command -v sha1 > /dev/null - then - local_cmd_sum="sha1 -q %s" - elif - command -v sha1sum > /dev/null - then - local_cmd_sum="sha1sum %s | awk '{print \$1}'" - fi - ;; - sha256) - if command -v sha256 > /dev/null - then - local_cmd_sum="sha256 -q %s" - elif - command -v sha256sum > /dev/null - then - local_cmd_sum="sha256sum %s | awk '{print \$1}'" - fi - ;; - *) - # we arrive here only if --sum is given with unknown format prefix - echo "unknown hash format: $sum_hash" >&2 - exit 1 - ;; - esac - - if [ -z "$local_cmd_sum" ] - then - echo 'local checksum verification failed, no usable utility' >&2 - exit 1 - fi - fi - - # shellcheck disable=SC2059 - echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" - - echo "if [ \"\$sum_is\" != '$sum_should' ]; then" - - echo "echo 'local download checksum mismatch' >&2" - - echo "rm -f \"\$download_tmp\"" - - echo 'exit 1; fi' -fi +printf "$cmd > %s\n" \ + "$url" \ + "$tmp" if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -146,10 +47,12 @@ else target_host="$__target_host" fi -# shellcheck disable=SC2016 -printf '%s "$download_tmp" %s:%s\n' \ +printf '%s %s %s:%s\n' \ "$__remote_copy" \ + "$tmp" \ "$target_host" \ "$dst" -echo "rm -f \"\$download_tmp\"" +echo "rm -f '$tmp'" + +echo 'downloaded' > "$__messages_out" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index e49bcec3..029a0801 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,51 +6,17 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" + cmd="$( cat "$__object/explorer/remote_cmd" )" url="$( cat "$__object/parameter/url" )" - if [ -f "$__object/parameter/destination" ] - then - dst="$( cat "$__object/parameter/destination" )" - else - dst="/$__object_id" - fi + dst="/$__object_id" - echo "download_tmp=\"\$( mktemp )\"" + printf "$cmd > %s\n" \ + "$url" \ + "$dst" - # shellcheck disable=SC2059 - printf "$cmd_get > \"\$download_tmp\"\n" "$url" - - if [ -f "$__object/parameter/sum" ] - then - sum_should="$( cat "$__object/parameter/sum" )" - - if [ -f "$__object/parameter/cmd-sum" ] - then - remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" - else - remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" - - if echo "$sum_should" | grep -Fq ':' - then - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" - fi - fi - - # shellcheck disable=SC2059 - echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" - - echo "if [ \"\$sum_is\" != '$sum_should' ]; then" - - echo "echo 'remote download checksum mismatch' >&2" - - echo "rm -f \"\$download_tmp\"" - - echo 'exit 1; fi' - fi - - echo "mv \"\$download_tmp\" '$dst'" + echo 'downloaded' > "$__messages_out" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index c16510a9..a1278cfb 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,7 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``curl``, ``fetch`` or ``wget``. +By default type will try to use ``wget``, ``curl`` or ``fetch``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,8 +16,6 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). -To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. - REQUIRED PARAMETERS ------------------- @@ -27,29 +25,14 @@ url OPTIONAL PARAMETERS ------------------- -destination - Downloaded file's destination in target. If unset, ``$__object_id`` is used. - sum - Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. - - Type tries to detect hash format with regexes, but prefixes - ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. - - Checksum have two purposes - state check and post-download verification. - In state check, if destination checksum mismatches, then content of URL - will be downloaded to temporary file. If downloaded temporary file's - checksum matches, then it will be moved to destination (overwritten). - - For local downloads it is expected that usable utilities for checksum - calculation exist in the system. + Checksum is used to decide if existing destination file must be redownloaded. + By default output of ``cksum`` without filename is expected. + Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. download - If ``local`` (default), then file is downloaded to local storage and copied - to target host. If ``remote``, then download happens in target. - - For local downloads it is expected that usable utilities for downloading - exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. + If ``local`` (default), then download file to local storage and copy + it to target host. If ``remote``, then download happens in target. cmd-get Command used for downloading. @@ -79,7 +62,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum 46da3021ca9eace277115ec9106c5b46 + --sum md5:46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 3d4c498b..7ec8d86d 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" +if grep -Eq '^wget' "$__object/explorer/remote_cmd" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index e809ef78..d69e083e 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,6 +1,5 @@ +sum cmd-get cmd-sum -destination download onchange -sum diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index d376c09f..9be3c575 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|gentoo|redhat|suse|ubuntu) + alpine|centos|fedora|redhat|suse|gentoo) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 0d944482..d145c4c3 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*|11*) + 10*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 638a99e0..6394f629 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*|11*) + 10*) __package_apt certbot ;; diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index 79c0d9d3..fbfca330 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,24 +81,12 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get $apt_opts update > /dev/null 2>&1 || true +then echo apt-get update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index ca9aa45a..3f88f6bc 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,7 +37,6 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 - # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index a10c16d3..803468b5 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,19 +41,7 @@ fi case "$type" in yum) ;; apt) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - - echo "apt-get --quiet $apt_opts update" + echo "apt-get --quiet update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e9f3c131..e36ded2f 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,104 +1,39 @@ #!/bin/sh -e - -if ! command -v rsync > /dev/null -then - echo 'rsync is missing in local machine' >&2 - exit 1 -fi - -src="$( cat "$__object/parameter/source" )" - -if [ ! -e "$src" ] -then - echo "$src not found" >&2 - exit 1 -fi - -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi - -# if source is directory, then make sure that -# source and destination are ending with slash, -# because this is what you almost always want when -# rsyncing two directories. - -if [ -d "$src" ] -then - if ! echo "$src" | grep -Eq '/$' - then - src="$src/" - fi - - if ! echo "$dst" | grep -Eq '/$' - then - dst="$dst/" - fi -fi - -remote_user="$( cat "$__object/parameter/remote-user" )" - -options="$( cat "$__object/parameter/options" )" - -if [ -f "$__object/parameter/option" ] -then - while read -r l - do - # there's a limitation in argparse: value can't begin with '-'. - # to workaround this, let's prefix opts with '\' in manifest and remove here. - # read more about argparse issue: https://bugs.python.org/issue9334 - - options="$options $( echo "$l" | sed 's/\\//g' )" - done \ - < "$__object/parameter/option" -fi - -if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] -then - options="$options --chown=" - - if [ -f "$__object/parameter/owner" ] - then - owner="$( cat "$__object/parameter/owner" )" - options="$options$owner" - fi - - if [ -f "$__object/parameter/group" ] - then - group="$( cat "$__object/parameter/group" )" - options="$options:$group" - fi -fi - -if [ -f "$__object/parameter/mode" ] -then - mode="$( cat "$__object/parameter/mode" )" - options="$options --chmod=$mode" -fi - -# IMPORTANT # -# 1. we first dry-run rsync with change summary to find out -# if there are any changes and code generation is needed. -# 2. normally, to get current state or target host, we run -# such operations in type explorers, but that's not -# possible due to how rsync works. -# 3. redirecting output of dry-run to stderr to ease debugging. -# 4. to understand how that cryptic regex works, please -# open rsync manpage and read about --itemize-changes. +# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# -export RSYNC_RSH="$__remote_exec" +source=$(cat "$__object/parameter/source") +remote_user=$(cat "$__object/parameter/remote-user") -# shellcheck disable=SC2086 -if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ - | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 -then - exit 0 +if [ -f "$__object/parameter/destination" ]; then + destination=$(cat "$__object/parameter/destination") +else + destination="/$__object_id" fi -echo "export RSYNC_RSH='$__remote_exec'" +set -- +if [ -f "$__object/parameter/rsync-opts" ]; then + while read -r opts; do + set -- "$@" "--$opts" + done < "$__object/parameter/rsync-opts" +fi -echo "rsync $options $src $remote_user@$__target_host:$dst" +echo rsync -a \ + --no-owner --no-group \ + -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__rsync/gencode-remote similarity index 56% rename from cdist/conf/type/__debconf_set_selections/manifest rename to cdist/conf/type/__rsync/gencode-remote index 0f4fb2e2..074246af 100755 --- a/cdist/conf/type/__debconf_set_selections/manifest +++ b/cdist/conf/type/__rsync/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2015 Dominique Roux (dominique.roux4 at gmail.com) # # This file is part of cdist. # @@ -18,4 +18,20 @@ # along with cdist. If not, see . # -__package_apt debconf +if [ -f "$__object/parameter/destination" ]; then + destination=$(cat "$__object/parameter/destination") +else + destination="/$__object_id" +fi + +ownergroup="" +if [ -f "$__object/parameter/owner" ]; then + ownergroup=$(cat "$__object/parameter/owner") +fi +if [ -f "$__object/parameter/group" ]; then + ownergroup="${ownergroup}:$(cat "$__object/parameter/group")" +fi + +if [ "$ownergroup" ]; then + echo chown -R "$ownergroup" "$destination" +fi diff --git a/cdist/conf/type/__rsync/man.rst b/cdist/conf/type/__rsync/man.rst index 88019c92..94b06d63 100644 --- a/cdist/conf/type/__rsync/man.rst +++ b/cdist/conf/type/__rsync/man.rst @@ -3,73 +3,112 @@ cdist-type__rsync(7) NAME ---- -cdist-type__rsync - Mirror directories using ``rsync`` +cdist-type__rsync - Mirror directories using rsync DESCRIPTION ----------- -The purpose of this type is to bring power of ``rsync`` into ``cdist``. +WARNING: This type is of BETA quality: + +- it has not been tested widely +- interfaces *may* change +- if there is a better approach to solve the problem -> the type may even vanish + +If you are fine with these constraints, please read on. + + +This cdist type allows you to mirror local directories to the +target host using rsync. Rsync will be installed in the manifest of the type. +If group or owner are giveng, a recursive chown will be executed on the +target host. + +A slash will be appended to the source directory so that only the contents +of the directory are taken and not the directory name itself. REQUIRED PARAMETERS ------------------- source - Source directory in local machine. - If source is directory, slash (``/``) will be added to source and destination paths. + Where to take files from OPTIONAL PARAMETERS ------------------- -destination - Destination directory. Defaults to ``$__object_id``. +group + Group to chgrp to. owner - Will be passed to ``rsync`` as ``--chown=OWNER``. - Read ``rsync(1)`` for more details. + User to chown to. -group - Will be passed to ``rsync`` as ``--chown=:GROUP``. - Read ``rsync(1)`` for more details. - -mode - Will be passed to ``rsync`` as ``--chmod=MODE``. - Read ``rsync(1)`` for more details. - -options - Defaults to ``--recursive --links --perms --times``. - Due to `bug in Python's argparse`_, value must be prefixed with ``\``. +destination + Use this as the base destination instead of the object id remote-user - Defaults to ``root``. + Use this user instead of the default "root" for rsync operations. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -option - Pass additional options to ``rsync``. - See ``rsync(1)`` for all possible options. - Due to `bug in Python's argparse`_, value must be prefixed with ``\``. +rsync-opts + Use this option to give rsync options with. + See rsync(1) for available options. + Only "--" options are supported. + Write the options without the beginning "--" + Can be specified multiple times. + + +MESSAGES +-------- +NONE EXAMPLES -------- + .. code-block:: sh - __rsync /var/www/example.com \ - --owner root \ - --group www-data \ - --mode 'D750,F640' \ - --source "$__files/example.com/www" + # You can use any source directory + __rsync /tmp/testdir \ + --source /etc + + # Use source from type + __rsync /etc \ + --source "$__type/files/package" + + # Allow multiple __rsync objects to write to the same dir + __rsync mystuff \ + --destination /usr/local/bin \ + --source "$__type/files/package" + + __rsync otherstuff \ + --destination /usr/local/bin \ + --source "$__type/files/package2" + + # Use rsync option --exclude + __rsync /tmp/testdir \ + --source /etc \ + --rsync-opts exclude=sshd_conf + + # Use rsync with multiple options --exclude --dry-run + __rsync /tmp/testing \ + --source /home/tester \ + --rsync-opts exclude=id_rsa \ + --rsync-opts dry-run + + +SEE ALSO +-------- +:strong:`rsync`\ (1) AUTHORS ------- -Ander Punnar +Nico Schottelius COPYING ------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. +Copyright \(C) 2015 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 64fa804e..9bd44c6d 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,3 +1,21 @@ #!/bin/sh -e +# +# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options deleted file mode 100644 index d967b110..00000000 --- a/cdist/conf/type/__rsync/parameter/default/options +++ /dev/null @@ -1 +0,0 @@ ---recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index 833e9bbe..ac2b2390 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,6 +1,4 @@ destination -group -mode -options owner +group remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index 01925a15..fdb7cd88 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -option +rsync-opts diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file deleted file mode 100755 index ec3d0fe8..00000000 --- a/cdist/conf/type/__sed/explorer/file +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -if [ ! -e "$file" ] -then - echo "$file does not exist" >&2 - exit 1 -fi - -cat "$file" diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote deleted file mode 100755 index f99c5a88..00000000 --- a/cdist/conf/type/__sed/gencode-remote +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -script="$( cat "$__object/parameter/script" )" - -if [ "$script" = '-' ] -then - script="$( cat "$__object/stdin" )" -fi - -# since stdin is not available in explorer, we pull file from target with explorer - -file_from_target="$__object/explorer/file" - -sed_cmd='sed' - -if [ -f "$__object/parameter/regexp-extended" ] -then - sed_cmd="$sed_cmd -E" -fi - -# do sed dry run, diff result and if no change, then there's nothing to do -# also redirect diff's output to stderr for debugging purposes - -if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 -then - exit 0 -fi - -# we can't use -i, because it's not posix, so we fly with tempfile and cp -# and we use cp because we want to preserve destination file's attributes - -# shellcheck disable=SC2016 -echo 'tmp="$__object/tempfile"' - -echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - -echo "$script" - -echo 'EOF' - -echo "cp \"\$tmp\" '$file'" - -# shellcheck disable=SC2016 -echo 'rm -f "$tmp"' - -echo 'change' >> "$__messages_out" - -if [ -f "$__object/parameter/onchange" ] -then - cat "$__object/parameter/onchange" -fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst deleted file mode 100644 index 86789363..00000000 --- a/cdist/conf/type/__sed/man.rst +++ /dev/null @@ -1,57 +0,0 @@ -cdist-type__sed(7) -================== - -NAME ----- -cdist-type__sed - Transform text files with ``sed`` - - -DESCRIPTION ------------ -Transform text files with ``sed``. - - -REQUIRED MULTIPLE PARAMETERS ----------------------------- -script - ``sed`` script. - If ``-`` then the script is read from ``stdin``. - - -OPTIONAL PARAMETERS -------------------- -file - Path to the file. Defaults to ``$__object_id``. - -onchange - Execute this command if ``sed`` changes file. - - -BOOLEAN PARAMETERS ------------------- -regexp-extended - Use extended regular expressions in the script. - Might not be supported with every ``sed`` version. - - -EXAMPLES --------- - -.. code-block:: sh - - __sed /tmp/foobar --script 's/foo/bar/' - - echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - - - -AUTHORS -------- -Ander Punnar - - -COPYING -------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean deleted file mode 100644 index 1ad75c5d..00000000 --- a/cdist/conf/type/__sed/parameter/boolean +++ /dev/null @@ -1 +0,0 @@ -regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional deleted file mode 100644 index fa86f917..00000000 --- a/cdist/conf/type/__sed/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -file -onchange diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple deleted file mode 100644 index 84f7e31d..00000000 --- a/cdist/conf/type/__sed/parameter/required_multiple +++ /dev/null @@ -1 +0,0 @@ -script diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst index b0b0a2e9..0b547804 100644 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ b/cdist/conf/type/__snakeoil_cert/man.rst @@ -38,7 +38,6 @@ cert-path EXAMPLES -------- .. code-block:: sh - __snakeoil_cert localhost-rsa \ --common-name localhost \ --key-type rsa:4096 diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index 9694a64b..cec25746 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,7 +1,6 @@ #!/bin/sh -e # shellcheck disable=SC1090 -# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index bb1619a9..34aaca56 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ -| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +update-alternatives --display "$__object_id" 2>/dev/null \ + | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index d1087c75..6519e7c2 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/${__object_id:?}" ] + if [ ! -f "$altdir/$__object_id" ] then continue fi - link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" + link="$( awk 'NR==2' "$altdir/$__object_id" )" if [ -n "$link" ] then @@ -31,12 +31,9 @@ do fi done -if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] +if [ -z "$link" ] then - # NOTE: ignore error for dry-runs because a package providing the link - # might be managed by another cdist object (which wasn't executed, - # because dry run…). - echo "unable to get link for ${__object_id:?}" >&2 + echo "unable to get link for $__object_id" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 5cf4fa4b..fc304d5d 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,15 +1,11 @@ #!/bin/sh -e -path_is=$( - LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ - | awk '/link currently points to/ { print $5 }') +path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ + | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] +if [ -z "$path_is" ] then - # NOTE: ignore error for dry-runs because a package providing the - # alternative might be managed by another cdist object (which - # wasn't executed, because dry run…). - echo "unable to get current path for ${__object_id:?}" >&2 + echo "unable to get current path for $__object_id" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index b74a7ee8..59e015c5 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "${__object:?}/parameter/path" )" ] +if [ -f "$( cat "$__object/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e91ea78f..e393cdef 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,39 +18,37 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "${__object:?}/explorer/path_is" )" +path_is="$( cat "$__object/explorer/path_is" )" -path_should="$( cat "${__object:?}/parameter/path" )" +path_should="$( cat "$__object/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ - && [ -z "${__cdist_dry_run+dry run}" ] +if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name=${__object_id:?} +name="$__object_id" -if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" +alternatives="$( cat "$__object/explorer/alternatives" )" + +if ! echo "$alternatives" | grep -Fxq "$path_should" then - if [ -f "${__object:?}/parameter/install" ] + if [ ! -f "$__object/parameter/install" ] then - link="$( cat "${__object:?}/explorer/link" )" - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" - elif [ -z "${__cdist_dry_run+dry run}" ] - then - # NOTE: ignore error for dry-runs because a package providing the link - # to be installed might be managed by another cdist object (which - # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi + + link="$( cat "$__object/explorer/link" )" + + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" diff --git a/cdist/log.py b/cdist/log.py index 62e457fe..113f3b4c 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,27 +36,25 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') - logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(self, msg, *args, **kwargs): - self.log(logging.VERBOSE, msg, args, **kwargs) +def _verbose(msg, *args, **kwargs): + logging.log(logging.VERBOSE, msg, *args, **kwargs) -logging.Logger.verbose = _verbose - +logging.verbose = _verbose logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(self, msg, *args, **kwargs): - self.log(logging.TRACE, msg, *args, **kwargs) +def _trace(msg, *args, **kwargs): + logging.log(logging.TRACE, msg, *args, **kwargs) -logging.Logger.trace = _trace +logging.trace = _trace class CdistFormatter(logging.Formatter): diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index ddbe4933..eca4cf13 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,98 +20,36 @@ # import logging -import sys -from datetime import datetime log = logging.getLogger("scan") -def run(scan, args): - # We run each component in a separate process since they - # must not block on each other. +# define this outside of the class to not handle scapy import errors by default +def commandline(args): + log.debug(args) + + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + print('cdist scan requires scapy to be installed') + processes = [] + if not args.mode: + # By default scan and trigger, but do not call any action + args.mode = ['scan', 'trigger', ] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interface, - sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces) t.start() processes.append(t) log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner( - autoconfigure='config' in args.mode, - interfaces=args.interface, - name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interfaces, args=args) s.start() processes.append(s) log.debug("Scanner started") for process in processes: process.join() - - -def list(scan, args): - s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) - hosts = s.list() - - # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + - # 6 colons. - ipv6_max_size = 8 * 4 + 10 - date_max_size = len(datetime.now().strftime(scan.datetime_format)) - name_max_size = 25 - - print("{} | {} | {} | {}".format( - 'name'.ljust(name_max_size), - 'address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size), - 'last configured'.ljust(date_max_size))) - print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) - for host in hosts: - last_seen = host.last_seen() - if last_seen: - last_seen = last_seen.strftime(scan.datetime_format) - else: - last_seen = '-' - - last_configured = host.last_configured() - if last_configured is not None: - last_configured = last_configured.strftime(scan.datetime_format) - else: - last_configured = '-' - - print("{} | {} | {} | {}".format( - host.name(default='-').ljust(name_max_size), - host.address().ljust(ipv6_max_size), - last_seen.ljust(date_max_size), - last_configured.ljust(date_max_size))) - - -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Set default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - - if 'config' in args.mode and args.name_mapper is None: - print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) - sys.exit(1) - - # Print known hosts and exit is --list is specified - do not start - # the scanner. - if args.list: - list(scan, args) - else: - run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 4a20f511..faee8a56 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,6 +19,38 @@ # # +# +# Interface to be implemented: +# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated +# scan: scan / listen for icmp6 replies +# trigger: send trigger to multicast +# config: configure newly detected hosts +# install: install newly detected hosts +# +# Scanner logic +# - save results to configdir: +# basedir = ~/.cdist/scan/ +# last_seen = ~/.cdist/scan//last_seen -- record unix time +# or similar +# last_configured = ~/.cdist/scan//last_configured -- record +# unix time or similar +# last_installed = ~/.cdist/scan//last_configured -- record +# unix time or similar +# +# +# +# +# cdist scan --list +# Show all known hosts including last seen flag +# +# Logic for reconfiguration: +# +# - record when configured last time +# - introduce a parameter --reconfigure-after that takes time argument +# - reconfigure if a) host alive and b) reconfigure-after time passed +# + + from multiprocessing import Process import os import logging @@ -29,84 +61,7 @@ import datetime import cdist.config -logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") -datetime_format = '%Y-%m-%d %H:%M:%S' - - -class Host(object): - def __init__(self, addr, outdir, name_mapper=None): - self.addr = addr - self.workdir = os.path.join(outdir, addr) - self.name_mapper = name_mapper - - os.makedirs(self.workdir, exist_ok=True) - - def __get(self, key, default=None): - fname = os.path.join(self.workdir, key) - value = default - if os.path.isfile(fname): - with open(fname, "r") as fd: - value = fd.readline() - return value - - def __set(self, key, value): - fname = os.path.join(self.workdir, key) - with open(fname, "w") as fd: - fd.write(f"{value}") - - def name(self, default=None): - if self.name_mapper is None: - return default - - fpath = os.path.join(os.getcwd(), self.name_mapper) - if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: - value = out.stdout.decode() - return (default if len(value) == 0 else value) - else: - return default - - def address(self): - return self.addr - - def last_seen(self, default=None): - raw = self.__get('last_seen') - if raw: - return datetime.datetime.strptime(raw, datetime_format) - else: - return default - - def last_configured(self, default=None): - raw = self.__get('last_configured') - if raw: - return datetime.datetime.strptime(raw, datetime_format) - else: - return default - - def seen(self): - now = datetime.datetime.now().strftime(datetime_format) - self.__set('last_seen', now) - - # XXX: There's no easy way to use the config module without feeding it with - # CLI args. Might as well call everything from scratch! - def configure(self): - target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target] - - fname = os.path.join(self.workdir, 'last_configuration_log') - with open(fname, "w") as fd: - log.debug("Executing: %s", cmd) - completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) - if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for \ - details.", cmd, completed_process.returncode, fname) - - now = datetime.datetime.now().strftime(datetime_format) - self.__set('last_configured', now) class Trigger(object): @@ -114,14 +69,12 @@ class Trigger(object): Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces, sleeptime, verbose=False): + def __init__(self, interfaces=None, verbose=False): self.interfaces = interfaces - - # Used by scapy / send in trigger/2. self.verbose = verbose - # Delay in seconds between sent ICMPv6EchoRequests. - self.sleeptime = sleeptime + # Wait 5 seconds before triggering again - FIXME: add parameter + self.sleeptime = 5 def start(self): self.processes = [] @@ -140,14 +93,9 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - try: - log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6( - dst="ff02::1%{}".format(interface) - ) / ICMPv6EchoRequest() - send(packet, verbose=self.verbose) - except Exception as e: - log.error("Could not send ICMPv6EchoRequest: %s", e) + packet = IPv6(dst="ff02::1{}".format(interface)) / ICMPv6EchoRequest() + log.debug("Sending request on %s", interface) + send(packet, verbose=self.verbose) class Scanner(object): @@ -155,62 +103,41 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, - name_mapper=None): + def __init__(self, interfaces=None, args=None, outdir=None): self.interfaces = interfaces - self.autoconfigure = autoconfigure - self.name_mapper = name_mapper - self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') - os.makedirs(self.outdir, exist_ok=True) - - self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) - if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), - host.address()) - else: - log.verbose("Host %s is alive", host.address()) + host = pkg['IPv6'].src + log.verbose("Host %s is alive", host) - host.seen() + dir = os.path.join(self.outdir, host) + fname = os.path.join(dir, "last_seen") - # Configure if needed. - if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + \ - self.config_delay < datetime.datetime.now(): - self.config(host) + now = datetime.datetime.now() - def list(self): - hosts = [] - for addr in os.listdir(self.outdir): - hosts.append(Host(addr, self.outdir, self.name_mapper)) + os.makedirs(dir, exist_ok=True) - return hosts + # FIXME: maybe adjust the format so we can easily parse again + with open(fname, "w") as fd: + fd.write(f"{now}\n") - def config(self, host): - if host.name() is None: - log.debug("config - could not resolve name for %s, aborting.", - host.address()) - return + def config(self): + """ + Configure a host - previous_config_process = self.running_configs.get(host.name()) - if previous_config_process is not None and \ - previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", - host.name()) + - Assume we are only called if necessary + - However we need to ensure to not run in parallel + - Maybe keep dict storing per host processes + - Save the result + - Save the output -> probably aligned to config mode - log.info("config - running against host %s (%s).", host.name(), - host.address()) - p = Process(target=host.configure()) - p.start() - self.running_configs[host.name()] = p + """ def start(self): self.process = Process(target=self.scan) @@ -221,9 +148,47 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - try: - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - except Exception as e: - log.error("Could not start listener: %s", e) + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + + +if __name__ == '__main__': + t = Trigger(interfaces=["wlan0"]) + t.start() + + # Scanner can listen on many interfaces at the same time + s = Scanner(interfaces=["wlan0"]) + s.scan() + + # Join back the trigger processes + t.join() + + # Test in my lan shows: + # [18:48] bridge:cdist% ls -1d fe80::* + # fe80::142d:f0a5:725b:1103 + # fe80::20d:b9ff:fe49:ac11 + # fe80::20d:b9ff:fe4c:547d + # fe80::219:d2ff:feb2:2e12 + # fe80::21b:fcff:feee:f446 + # fe80::21b:fcff:feee:f45c + # fe80::21b:fcff:feee:f4b1 + # fe80::21b:fcff:feee:f4ba + # fe80::21b:fcff:feee:f4bc + # fe80::21b:fcff:feee:f4c1 + # fe80::21d:72ff:fe86:46b + # fe80::42b0:34ff:fe6f:f6f0 + # fe80::42b0:34ff:fe6f:f863 + # fe80::42b0:34ff:fe6f:f9b2 + # fe80::4a5d:60ff:fea1:e55f + # fe80::77a3:5e3f:82cc:f2e5 + # fe80::9e93:4eff:fe6c:c1f4 + # fe80::ba69:f4ff:fec5:6041 + # fe80::ba69:f4ff:fec5:8db7 + # fe80::bad8:12ff:fe65:313d + # fe80::bad8:12ff:fe65:d9b1 + # fe80::ce2d:e0ff:fed4:2611 + # fe80::ce32:e5ff:fe79:7ea7 + # fe80::d66d:6dff:fe33:e00 + # fe80::e2ff:f7ff:fe00:20e6 + # fe80::f29f:c2ff:fe7c:275e diff --git a/docs/changelog b/docs/changelog index 6f717cf4..97ea204b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,29 +2,6 @@ Changelog --------- next: - * Explorer machine_type: Rewrite (Dennis Camera) - * New type: __sed (Ander Punnar) - * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __apt_source: Fix complaint about suite change (Matthias Stecher) - * Type __package_apt: Fix complaint about suite change (Matthias Stecher) - * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) - * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) - -6.9.8: 2021-08-24 - * Type __rsync: Rewrite (Ander Punnar) - * New type: __apt_pin (Daniel Fancsali) - * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) - * Core: Fix logging bug (Dennis Camera) - * Build: Improve Makefile compatibility (Evilham) - * Type __filesystem: Support ubuntu (Joachim Desroches) - * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) - * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) - * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) - * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) - * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) - -6.9.7: 2021-07-10 * New type: __postgres_conf (Beni Ruef, Dennis Camera) * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) @@ -32,11 +9,6 @@ next: * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) * Type __git: Fix group explorer (Ander Punnar) * Type __pyvenv: Fix group explorer (Dennis Camera) - * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) - * Type __debconf_set_selections: Add state explorer (Dennis Camera) - * Core: Implement usable cdist scan (Timothée Floure) - * New type: __snakeoil_cert (Ander Punnar) - * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) 6.9.6: 2021-04-20 * Type __pyvenv: Fix user example in man page (Dennis Camera) @@ -165,7 +137,7 @@ next: * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -230,7 +202,7 @@ next: * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1089,9 +1061,9 @@ next: * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 390ab9ec..18863145 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx with the rtd theme (for building html docs and/or the man pages) + * sphinx (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst deleted file mode 100644 index 86b7fab6..00000000 --- a/docs/src/cdist-scan.rst +++ /dev/null @@ -1,84 +0,0 @@ -Scan -===== - -Description ------------ -Runs cdist as a daemon that discover/watch on hosts and reconfigure them -periodically. It is especially useful in netboot-based environment where hosts -boot unconfigured, and to ensure your infrastructure stays in sync with your -configuration. - -This feature is still consider to be in **beta** stage, and only operate on -IPv6 (including link-local). - -Usage (Examples) ----------------- - -Discover hosts on local network and configure those whose name is resolved by -the name mapper script. - -.. code-block:: sh - - $ cdist scan --beta --interface eth0 \ - --mode scan --name-mapper path/to/script \ - --mode trigger --mode config - -List known hosts and exit. - -.. code-block:: sh - - $ cdist scan --beta --list --name-mapper path/to/script - -Please refer to `cdist(1)` for a detailed list of parameters. - -Modes ------ - -The scanner has 3 modes that can be independently toggled. If the `--mode` -parameter is not specified, only `tigger` and `scan` are enabled (= hosts are -not configured). - -trigger - Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to - trigger detection by the `scan` module. - -scan - Watch for incoming ICMPv6 replies and optionally configure detected hosts. - -config - Enable configuration of hosts detected by `scan`. - -Name Mapper Script ------------------- - -The name mapper script takes an IPv6 address as first argument and writes the -resolved name to stdout - if any. The script must be executable. - -Simplest script: - -.. code-block:: sh - - #!/bin/sh - - case "$1" in - "fe80::20d:b9ff:fe57:3524") - printf "my-host-01" - ;; - "fe80::7603:bdff:fe05:89bb") - printf "my-host-02" - ;; - esac - -Resolving name from `PTR` DNS record: - -.. code-block:: sh - - #!/bin/sh - - for cmd in dig sed; do - if ! command -v $cmd > /dev/null; then - exit 1 - fi - done - - dig +short -x "$1" | sed -e 's/.$//' diff --git a/docs/src/conf.py b/docs/src/conf.py index a3dfafca..47765413 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2021' +copyright = 'ungleich GmbH 2020' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/src/index.rst b/docs/src/index.rst index 369d5309..31c044dc 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,7 +34,6 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos - cdist-scan cdist-integration cdist-reference cdist-best-practice diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 599ec3b7..0ecb4a61 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,9 +88,6 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] - cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] - [-d CONFIG_DELAY] [-t TRIGGER_DELAY] - DESCRIPTION ----------- @@ -644,31 +641,6 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. -SCAN ----- - -Runs cdist as a daemon that discover/watch on hosts and reconfigure them -periodically. - -**-I INTERFACE, --interfaces INTERFACE** - Interface to listen on. Can be specified multiple times. - -**-m MODE, --mode MODE** - Scanner components to enable. Can be specified multiple time to enable more - than one component. Supported modes are: scan, trigger and config. Defaults - to tiggger and scan. - -**--name-mapper PATH_TO_SCRIPT** - Path to script used to resolve a remote host name from an IPv6 address. - -**--list** - List known hosts and exit. - -**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** - How long (seconds) to wait before reconfiguring after last try (config mode only). - -**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** - How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION -------------