diff --git a/Makefile b/Makefile index 89286310..3712511c 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,9 @@ DOCS_SRC_DIR=./docs/src SPEECHDIR=./docs/speeches TYPEDIR=./cdist/conf/type -SPHINXM=$(MAKE) -C $(DOCS_SRC_DIR) man -SPHINXH=$(MAKE) -C $(DOCS_SRC_DIR) html -SPHINXC=$(MAKE) -C $(DOCS_SRC_DIR) clean +SPHINXM=make -C $(DOCS_SRC_DIR) man +SPHINXH=make -C $(DOCS_SRC_DIR) html +SPHINXC=make -C $(DOCS_SRC_DIR) clean ################################################################################ # Manpages diff --git a/README.md b/README.md index a468dd86..de6901c7 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,8 @@ For community-maintained types there is ## Participating -IRC: ``#cdist`` @ [libera](https://libera.chat) +IRC: ``#cdist`` @ freenode Matrix: ``#cdist:ungleich.ch`` -Matrix and IRC are bridged. +Mattermost: https://chat.ungleich.ch/ungleich/channels/cdist diff --git a/bin/cdist b/bin/cdist index adb06a8d..ddaffa7f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -72,11 +72,9 @@ def commandline(): if __name__ == "__main__": - if sys.version_info[:3] < cdist.MIN_SUPPORTED_PYTHON_VERSION: - print( - 'Python >= {} is required on the source host.'.format( - ".".join(map(str, cdist.MIN_SUPPORTED_PYTHON_VERSION))), - file=sys.stderr) + if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: + print('Python >= {} is required on the source host.'.format( + cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) sys.exit(1) exit_code = 0 diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index cadddae7..0380b3f8 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. diff --git a/cdist/__init__.py b/cdist/__init__.py index 31d49889..44366cd0 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -64,7 +64,7 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" -MIN_SUPPORTED_PYTHON_VERSION = (3, 5) +MIN_SUPPORTED_PYTHON_VERSION = '3.5' class Error(Exception): diff --git a/cdist/argparse.py b/cdist/argparse.py index f17315e7..88759d7b 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -485,31 +485,19 @@ def get_parsers(): parser['scan'].add_argument( '-m', '--mode', help='Which modes should run', action='append', default=[], - choices=['scan', 'trigger', 'config']) - parser['scan'].add_argument( - '--list', - action='store_true', - help='List the known hosts and exit') + choices=['scan', 'trigger']) parser['scan'].add_argument( '--config', action='store_true', help='Try to configure detected hosts') parser['scan'].add_argument( - '-I', '--interface', - action='append', default=[], required=True, + '-I', '--interfaces', + action='append', default=[], help='On which interfaces to scan/trigger') parser['scan'].add_argument( - '--name-mapper', - action='store', default=None, - help='Map addresses to names, required for config mode') - parser['scan'].add_argument( - '-d', '--config-delay', - action='store', default=3600, type=int, - help='How long (seconds) to wait before reconfiguring after last try') - parser['scan'].add_argument( - '-t', '--trigger-delay', - action='store', default=5, type=int, - help='How long (seconds) to wait between ICMPv6 echo requests') + '-d', '--delay', + action='store', default=3600, + help='How long to wait before reconfiguring after last try') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) for p in parser: @@ -545,10 +533,10 @@ def parse_and_configure(argv, singleton=True): log = logging.getLogger("cdist") - log.verbose("version %s", cdist.VERSION) - log.trace('command line args: %s', cfg.command_line_args) - log.trace('configuration: %s', cfg.get_config()) - log.trace('configured args: %s', args) + log.verbose("version %s" % cdist.VERSION) + log.trace('command line args: {}'.format(cfg.command_line_args)) + log.trace('configuration: {}'.format(cfg.get_config())) + log.trace('configured args: {}'.format(args)) check_beta(vars(args)) diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index c9fb5cdf..26bb8e3d 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - awk '{printf("%s\n", $(NF-1))}' /etc/cp-release - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index 7279a9c2..b1009627 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - cat /etc/cp-release - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 1f91cc40..82ff9977 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - echo "CheckPoint" - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 0bb9f7fe..5ebfff1a 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index 00646c75..1c84f4d7 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,6 +1,8 @@ -#!/bin/sh -e +#!/bin/sh # -# 2021 Dennis Camera (cdist at dtnr.ch) +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# 2020 Evilham (contact at evilham.com) # # This file is part of cdist. # @@ -17,1019 +19,91 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer tries to determine what type of machine the target to be -# configured is (container, virtual machine, bare-metal). -# -# It will print one line for each layer it can detect. -# The format of all lines is: TYPE[ VERB VENDOR] -# -# VERB does not have a special meaning, it is just for better readability. -# -# e.g. -# container -# container on lxc -# virtual by kvm-spapr -# -# The third word of each line (except the first) can be composed of different -# parts concatenated with a `-' (minus) character, with each component being -# a specification of the previous, e.g.: -# - lxc-libvirt (LXC container, managed by libvirt) -# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) -# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) -# -# If this explorer cannot collect enough information about virtualization it -# will fall back to 'physical'. -# -# Add /sbin and /usr/sbin to the path so we can find system -# binaries like dmidecode. -PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' -PATH="/sbin:/usr/sbin:${PATH}" -export PATH +os=$("$__explorer/os") -arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') -uname_s=$(uname -s) - - -is_command() { command -v "$1" >/dev/null 2>&1; } - -files_same() { - # shellcheck disable=SC2012 - LC_ALL=C df -P "$1" "$2" 2>/dev/null | { - read -r _ # skip header line - read -r fs1 _ _ _ _ mp1 - read -r fs2 _ _ _ _ mp2 - test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 - } && - ls -1Ldi "$1" "$2" 2>/dev/null | { - read -r ino1 _ - read -r ino2 _ - test "${ino1}" = "${ino2}" || return 1 - } +vendor_string_to_machine_type() { + for vendor in vmware bochs kvm qemu virtualbox bhyve; do + if echo "${1}" | grep -q -i "${vendor}"; then + if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then + vendor="kvm" + fi + echo "virtual_by_${vendor}" + exit + fi + done } -is_oneof() ( - x=$1; shift - for y - do - test "${x}" = "${y}" || continue - return 0 - done - return 1 -) - -tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } - -# shellcheck disable=SC2086 -glob_exists() { set -- $1; test -e "$1"; } - -get_dmi_field() { - if is_oneof "${uname_s}" NetBSD - then - case $1 - in - (system-manufacturer) _mib=machdep.dmi.system-vendor ;; - (system-product-name) _mib=machdep.dmi.system-product ;; - (system-version|system-uuid) _mib=machdep.dmi.$1 ;; - (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; - (biod-release-date) _mib=machdep.dmi.bios-date ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - if is_command dmidecode - then - dmidecode -s "$1" - elif test -d "${dmi_sysfs-}" - then - case $1 - in - (system-manufacturer) _filename=sys_vendor ;; - (system-product-name) _filename=product_name ;; - (*) _filename=$(echo "$1" | tr - _) ;; - esac - if test -r "${dmi_sysfs-}/${_filename}" - then - cat "${dmi_sysfs}/${_filename}" - fi - unset _filename - elif test "${uname_s}" = OpenBSD - then - # NOTE: something similar to system-manufacutrer and system-product-name - # is available on OpenBSD in sysctl - case $1 - in - (system-manufacturer) _mib=hw.vendor ;; - (system-product-name) _mib=hw.product ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - return 1 -} - -has_cpuinfo() { test -e /proc/cpuinfo; } - -get_sysctl() { - is_command sysctl && sysctl -n "$1" 2>/dev/null -} - -detected_layer() { - test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" -} - - -# Check for chroot - -has_chroot_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' -} - -check_chroot_systemd() { - systemd-detect-virt -r -} - -has_chroot_debian_ischroot() { - is_command ischroot -} - -check_chroot_debian_ischroot() { - ischroot --default-false -} - -has_chroot_procfs() { - test -d /proc/ -} - -check_chroot_procfs() ( - is_chroot=false # default - if test -e /proc/1/root && ! files_same /proc/1/root / - then - is_chroot=true - fi - if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo - then - has_mountinfo=true - cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true - fi - - if ${is_chroot} - then - # try to determine where the chroot has been mounted - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # escape chroot and find mount point by inode - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - elif ${has_mountinfo} - then - while read -r mntid _ _ _ cmntpnt _ - do - read -r _ _ _ _ hmntpnt _ <<-EOF - $(grep -e "^$((mntid)) " /proc/1/mountinfo) - EOF - printf '%s\n' "${hmntpnt%${cmntpnt}}" - done /dev/null) && - case ${_ctengine} - in - (''|'none') - return 1 ;; - ('container-other') - return 0 ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - echo "${_ctengine}" ;; - esac -) - -has_ct_pid_1() { - test -r /run/systemd/container -o -r /proc/1/environ -} - -translate_container_name() { - case $1 - in - ('lxc') - echo lxc ;; - ('lxc-libvirt') - echo lxc-libvirt ;; - ('podman') - echo podman ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - return 1 ;; - esac - return 0 -} - -check_ct_pid_1() { - if test -r /run/systemd/container - then - translate_container_name "$(head -n1 /run/systemd/container)" \ - && return 0 - fi - - if test -r /proc/1/environ - then - translate_container_name "$( - LC_ALL=C tr '\000' '\n' /dev/null - then - # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 - echo wsl - elif test -d /var/.cagefs - then - # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs - # CageFS is not "really" a container, but it isn't a chroot either. - echo cagefs - elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status - then - # Linux-VServer - if grep -q -x -F 'VxID: 0' /proc/self/status - then - # host - return 1 - else - # guest - echo linux_vserver - fi - else - return 1 - fi -} - -check_ct_os_specific() ( - if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 - then - # FreeBSD jail - echo jail - return 0 - fi - - if is_command zonename && test "$(zonename)" != global - then - # Solaris zone - echo zone - return 0 - fi - - return 1 -) - - -# Check for hypervisor - -guess_hypervisor_from_cpu_model() { - case $1 - in - (*\ KVM\ *) - echo kvm ;; - (*\ QEMU\ *|QEMU\ *) - echo qemu ;; - (*) - return 1 ;; - esac -} - -has_vm_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' -} - -check_vm_systemd() ( - _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && - case ${_hypervisor} - in - (''|'none') - return 1 ;; - ('amazon') - echo aws ;; - ('bochs') - echo kvm ;; - ('microsoft') - # assumption - echo hyperv ;; - ('oracle') - echo virtualbox ;; - ('vm-other') - return 0 ;; - (*) - echo "${_hypervisor}" ;; - esac -) - -has_vm_cpuinfo() { has_cpuinfo; } - -check_vm_cpuinfo() { - if grep -q -F 'User Mode Linux' /proc/cpuinfo \ - || grep -q -F 'UML' /proc/cpuinfo - then - # User Mode Linux - echo uml - elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo - then - # IBM PowerVM Lx86 (Linux/x86 emulator) - echo powervm_lx86 - elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo - then - # IBM SystemZ (S/390) - if test -f /proc/sysinfo - then - if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo - then - echo kvm-s390 - return 0 - elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo - then - echo zvm - return 0 - elif grep -q -e '^LPAR ' /proc/sysinfo - then - echo zvm-lpar - return 0 - fi - fi - return 1 - else - if grep -q -e '^model name.*:' /proc/cpuinfo - then - sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ - | while read -r _cpu_model - do - guess_hypervisor_from_cpu_model "${_cpu_model}" - done \ - | sort \ - | uniq -c \ - | awk ' - { if ($1 > most_c) { most_c = $1; most_s = $2 } } - END { - if (most_s) print most_s - exit !most_s - }' \ - && return 0 - fi - return 1 - fi -} - -check_vm_arch_specific() { - case ${arch} - in - (ppc64|ppc64le) - # Check PPC64 LPAR, KVM - - # example /proc/cpuinfo line indicating 'not baremetal' - # platform : pSeries - # - # example /proc/ppc64/lparcfg systemtype line - # system_type=IBM pSeries (emulated by qemu) - - if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo - then - if test -e /proc/ppc64/lparcfg - then - # Assume LPAR, now detect shared or dedicated - if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg - then - echo powervm-shared - return 0 - else - echo powervm-dedicated - return 0 - fi - fi - fi - ;; - (sparc*) - # Check for SPARC LDoms - - if test -e /dev/mdesc - then - if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp - then - # control LDom - return 1 - else - # guest LDom - echo ldom-sparc - fi - - # MDPROP=/usr/lib/ldoms/mdprop.py - # if test -x "${MDPROP}" - # then - # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" - # then - # echo ldoms-root - # echo ldoms-io - # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" - # then - # echo ldoms-io - # fi - # fi - return 0 - fi - ;; - (i?86|x86*|amd64|i86pc) - # VMM CPUID flag denotes that this system is running under a VMM - if is_oneof "${uname_s}" Darwin - then - get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ - && return 0 - fi - if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)' /proc/cpuinfo - then - return 0 - fi - ;; - (ia64) - if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend - then - # PV-on-HVM drivers installed in a Xen guest - echo xen-hvm - return 0 - fi - ;; - esac - return 1 -} - -has_vm_dmi() { - # Check for various products in SMBIOS/DMI. - # Note that DMI doesn't exist on all architectures (only x86 and some ARM). - # On other architectures the $dmi variable will be empty. - - if test -d /sys/class/dmi/id/ - then - dmi_sysfs=/sys/class/dmi/id - elif test -d /sys/devices/virtual/dmi/id/ - then - dmi_sysfs=/sys/devices/virtual/dmi/id - fi - - # shellcheck disable=SC2015 - { - is_command dmidecode \ - && ( - # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line - dmi_out=$(dmidecode 2>&1) \ - && ! printf '%s\n' "${dmi_out}" \ - | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' - ) \ - || test -d "${dmi_sysfs}" - } -} - -check_vm_dmi() { - case $(get_dmi_field system-product-name) - in - (*.metal) - if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' - then - # AWS EC2 bare metal -> no virtualisation - return 1 - fi - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('Google Compute Engine') - echo gce - return 0 - ;; - ('RHEV Hypervisor') - # Red Hat Enterprise Virtualization - echo rhev - return 0 - ;; - ('KVM'|'Bochs'|'KVM Virtual Machine') - echo kvm - return 0 - ;; - ('Parallels Virtual Platform') - echo parallels - return 0 - ;; - ('VirtualBox') - echo virtualbox - return 0 - ;; - ('VMware Virtual Platform') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field system-manufacturer) - in - ('Alibaba'*) - case $(get_dmi_field system-product-name) - in - ('Alibaba Cloud ECS') - echo alibaba-ecs - ;; - (*) - echo alibaba - ;; - esac - return 0 - ;; - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('innotek GmbH'|'Oracle Corporation') - echo virtualbox - return 0 - ;; - ('Joyent') - if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' - then - # SmartOS KVM - echo kvm-smartdc_hvm - return 0 - fi - ;; - ('Microsoft Corporation'*) - if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' - then - if test -e /proc/irq/7/hyperv \ - || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null - then - echo hyperv - return 0 - fi - - case $(get_dmi_field system-version) - in - (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) - echo virtualpc - return 0 - ;; - (*) - echo hyperv - return 0 - ;; - esac - fi - ;; - ('Nutanix') - # Nutanix AHV. Similar to KVM. - if test "$(get_dmi_field system-product-name)" = 'AHV' - then - echo nutanix_ahv - return 0 - fi - ;; - ('oVirt') - echo ovirt - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('QEMU') - echo qemu - return 0 - ;; - ('VMware, Inc.') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field bios-vendor) - in - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('innotek GmbH') - echo virtualbox - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('Xen') - if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' - then - # AWS on Xen - echo aws-xen - return 0 - fi - ;; - esac - - return 1 -} - -check_vm_hyp_specific() { - if is_command vmware-checkvm && vmware-checkvm >/dev/null - then - # vmware-checkvm is provided by VMware's open-vm-tools - echo vmware - return 0 - elif test -d /proc/xen - then - test -r /proc/xen/capabilities && - if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null - then - # Xen dom0 - return 1 - else - # Xen domU - echo xen - return 0 - fi - fi - return 1 -} - -has_vm_dt() { - # OpenFirmware/Das U-Boot device-tree - test -d /proc/device-tree -} - -check_vm_dt() { - case ${arch} - in - (arm|aarch64) - if test -r /proc/device-tree/hypervisor/compatible - then - if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible - then - echo xen - return 0 - elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible - then - # e.g. VMware ESXi on ARM - echo vmware - return 0 - fi - fi - if glob_exists /proc/device-tree/fw-cfg@*/compatible - then - # qemu,fw-cfg-mmio - sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 - return 0 - fi - if grep -q -F 'dummy-virt' /proc/device-tree/compatible - then - echo lkvm - return 0 - fi - ;; - (ppc64*) - if test -d /proc/device-tree/hypervisor \ - && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible - then - # We are running as a spapr KVM guest on ppc64 - echo kvm-spapr - return 0 - fi - if test -r /proc/device-tree/ibm,partition-name \ - && test -r /proc/device-tree/hmc-managed\? \ - && test -r /proc/device-tree/chosen/qemu,graphic-width - then - echo powervm - fi - ;; - esac - return 1 -} - -has_vm_sys_hypervisor() { - test -d /sys/hypervisor/ -} - -check_vm_sys_hypervisor() { - test -r /sys/hypervisor/type && - case $(head -n1 /sys/hypervisor/type) - in - (xen) - # Ordinary kernel with pv_ops. There does not seem to be - # enough information at present to tell whether this is dom0 - # or domU. - echo xen - return 0 - ;; - esac - return 1 -} - -check_vm_os_specific() { - _hyp_generic=false - - case ${uname_s} - in - (Darwin) - if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ - && test "${hv_vmm_present}" -ne 0 - then - _hyp_generic=true - fi - ;; - (FreeBSD) - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest=$(get_sysctl kern.vm_guest | tolower) && - case ${vm_guest} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${vm_guest} - in - (hv) echo hyperv ;; - (vbox) echo virtualbox ;; - (*) echo "${vm_guest}" ;; - esac - return 0 - ;; - esac - ;; - (NetBSD) - machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && - case ${machdep_hv} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${machdep_hv} - in - (hyper-v) echo hyperv ;; - (xenhvm*) echo xen-hvm ;; - (xenpv*) echo xen-pv ;; - (xen*) echo xen ;; - (*) echo "${machdep_hv}" ;; - esac - return 0 - ;; - esac - ;; - (OpenBSD) - if is_command hostctl && glob_exists /dev/pvbus[0-9]* - then - for _pvbus in /dev/pvbus[0-9]* - do - _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue - case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') - in - (KVM) echo kvm ;; - (Hyper-V) echo hyperv ;; - (VMware) echo vmware ;; - (Xen) echo xen ;; - (bhyve) echo bhyve ;; - (OpenBSD) echo openbsd_vmm ;; - esac - return 0 - done - fi - ;; - (SunOS) - diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') - # NOTE: Don't use -e or -F in Solaris grep - if printf '%s\n' "${diag_conf}" | grep -q -i QEMU - then - echo qemu - return 0 - elif printf '%s\n' "${diag_conf}" | grep -q -i VMware - then - echo vmware - return 0 - fi - ;; - (Linux) - if is_command dmesg - then - while read -r line - do - case ${line} - in - ('Booting paravirtualized kernel on ') - case $(expr "${line}" : '.* kernel on \(.*\)') - in - ('Xen') - echo xen-pv; return 0 ;; - ('bare hardware') - return 1 ;; - esac - ;; - ('Hypervisor detected') - case $(expr "${line}" : '.*: *\(.*\)') - in - ('ACRN') - echo acrn ;; - ('Jailhouse') - echo jailhouse ;; - ('KVM') - echo kvm ;; - ('Microsoft Hyper-V') - echo hyperv ;; - ('VMware') - echo vmware ;; - ('Xen HVM') - echo xen-hvm ;; - ('Xen PV') - echo xen-pv ;; - esac - return 0 - ;; - (lpar:*' under hypervisor') - return 0 ;; - esac - done <<-EOF - $(dmesg 2>/dev/null | awk ' - /Booting paravirtualized kernel on / - /Hypervisor detected: / - /lpar: .* under hypervisor/ - ') - EOF - fi - esac - - # Try to guess hypervisor based on CPU model (sysctl hw.model if available) - if cpu_model=$(get_sysctl hw.model) - then - guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 - fi - - if ${_hyp_generic} - then - # cannot say which hypervisor, but one was detected - return 0 - else - return 1 - fi -} - -run_stage() { - if type "has_$1_$2" >/dev/null 2>&1 - then - "has_$1_$2" - else - true - fi \ - && "check_$1_$2" -} - - -# Execute chroot stages - -for stage in \ - procfs debian_ischroot systemd -do - chrootpnt=$(run_stage chroot ${stage}) || continue - is_chrooted=true - detected_layer 'chroot' - if test -n "${chrootpnt}" - then - echo chroot at "${chrootpnt}" - break - fi -done -if ${is_chrooted:-false} && test -z "${chrootpnt}" -then - # could determine chroot, but not its mount point - echo chroot -fi - - -# Execute container stages - -for stage in \ - systemd pid_1 cgroup files os_specific -do - ctengine=$(run_stage ct ${stage}) || continue - detected_layer 'container' - is_contained=true - if test -n "${ctengine}" - then - echo container on "${ctengine}" - break - fi -done -if ${is_contained:-false} && test -z "${ctengine}" -then - # none of the stages could determine the specific container engine, but - # we are running in some container. - echo container -fi - - -# Execute virtual machine / hypervisor stages - -for stage in \ - systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific -do - hypervisor=$(run_stage vm ${stage}) || continue - detected_layer 'virtual machine' - is_virtual=true - if test -n "${hypervisor}" - then - echo virtual by "${hypervisor}" - break - fi -done -if ${is_virtual:-false} && test -z "${hypervisor}" -then - # none of the stages could determine the specific hypervisor, but - # we are virtual. - echo virtual -fi - - -# Fallback - -detected_layer physical +case "$os" in + "freebsd") + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" + if [ -n "${vm_guest}" ]; then + if [ "${vm_guest}" = "none" ]; then + echo "physical" + exit + fi + echo "virtual_by_${vm_guest}" + exit + fi + ;; + + "openbsd") + # OpenBSD can also use the sysctl's: hw.vendor or hw.product. + # Note we can be reasonably sure about a machine being virtualised + # as long as we can identify the virtualisation technology. + # But not so much about it being physical... + # Patches are welcome / reach out if you have better ideas. + for sysctl in hw.vendor hw.product; do + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" + done + ;; + + *) + # Defaulting to linux for compatibility with previous cdist behaviour + + if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then + echo openvz + exit + fi + + if [ -e "/proc/1/environ" ] && + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then + echo lxc + exit + fi + + if [ -r /proc/cpuinfo ]; then + # this should only exist on virtual guest machines, + # tested on vmware, xen, kvm, bhyve + if grep -q "hypervisor" /proc/cpuinfo; then + # this file is aviable in xen guest systems + if [ -r /sys/hypervisor/type ]; then + if grep -q -i "xen" /sys/hypervisor/type; then + echo virtual_by_xen + exit + fi + else + for vendor_file in /sys/class/dmi/id/product_name \ + /sys/class/dmi/id/sys_vendor \ + /sys/class/dmi/id/chasis_vendor; do + if [ -r ${vendor_file} ]; then + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(cat "${vendor_file}")" + fi + done + fi + echo "virtual_by_unknown" + exit + else + echo "physical" + exit + fi + fi + ;; +esac + +echo "unknown" diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index c6d113cf..5ea15ada 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -1,9 +1,8 @@ -#!/bin/sh -e +#!/bin/sh # # 2014 Daniel Heule (hda at sfs.biz) # 2014 Thomas Oettli (otho at sfs.biz) # Copyright 2017, Philippe Gregoire -# 2020 Dennis Camera # # This file is part of cdist. # @@ -20,73 +19,24 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# Returns the amount of memory physically installed in the system, or if that -# cannot be determined the amount available to the operating system kernel, -# in kibibytes (kiB). +# -str2bytes() { - awk -F' ' ' - $2 == "B" || !$2 { print $1 } - $2 == "kB" { printf "%.f\n", ($1 * 1000) } - $2 == "MB" { printf "%.f\n", ($1 * 1000 * 1000) } - $2 == "GB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000) } - $2 == "TB" { printf "%.f\n", ($1 * 1000 * 1000 * 1000 * 1000) } - $2 == "kiB" { printf "%.f\n", ($1 * 1024) } - $2 == "MiB" { printf "%.f\n", ($1 * 1024 * 1024) } - $2 == "GiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024) } - $2 == "TiB" { printf "%.f\n", ($1 * 1024 * 1024 * 1024 * 1024) }' -} +# FIXME: other system types (not linux ...) -bytes2kib() { - awk '$0 > 0 { printf "%.f\n", ($0 / 1024) }' -} +os=$("$__explorer/os") +case "$os" in + "macosx") + echo "$(sysctl -n hw.memsize)/1024" | bc + ;; + *"bsd") + PATH=$(getconf PATH) + echo "$(sysctl -n hw.physmem) / 1048576" | bc + ;; -case $(uname -s) -in - (Darwin) - sysctl -n hw.memsize | bytes2kib - ;; - (FreeBSD) - sysctl -n hw.realmem | bytes2kib - ;; - (NetBSD|OpenBSD) - # NOTE: This reports "usable" memory, not physically installed memory. - command -p sysctl -n hw.physmem | bytes2kib - ;; - (SunOS) - # Make sure that awk from xpg4 is used for the scripts to work - export PATH="/usr/xpg4/bin:${PATH}" - prtconf \ - | awk -F ': ' ' - $1 == "Memory size" { sub(/Megabytes/, "MiB", $2); print $2 } - /^$/ { exit }' \ - | str2bytes \ - | bytes2kib - ;; - (Linux) - if test -d /sys/devices/system/memory - then - # Use memory blocks if the architecture (e.g. x86, PPC64, s390) - # supports them (they denote physical memory) - num_mem_blocks=$(cat /sys/devices/system/memory/memory[0-9]*/state | grep -cxF online) - mem_block_size=$(cat /sys/devices/system/memory/block_size_bytes) - - echo $((num_mem_blocks * 0x$mem_block_size)) | bytes2kib && exit - fi - if test -r /proc/meminfo - then - # Fall back to meminfo file on other architectures (e.g. ARM, MIPS, - # PowerPC) - # NOTE: This is "usable" memory, not physically installed memory. - awk -F ': +' '$1 == "MemTotal" { sub(/B$/, "iB", $2); print $2 }' /proc/meminfo \ - | str2bytes \ - | bytes2kib - fi - ;; - (*) - printf "Your kernel (%s) is currently not supported by the memory explorer\n" "$(uname -s)" >&2 - printf "Please contribute an implementation for it if you can.\n" >&2 - exit 1 - ;; + *) + if [ -r /proc/meminfo ]; then + grep "MemTotal:" /proc/meminfo | awk '{print $2}' + fi + ;; esac diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index b9232ee4..46d87f3e 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,13 +116,6 @@ if [ -f /etc/slackware-version ]; then exit 0 fi -# Appliances - -if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then - echo checkpoint - exit 0 -fi - uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index ec85046f..6489446b 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,9 +34,5 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release -elif test -f /etc/cp-release -then - # Checkpoint firewall or management (actually linux based) - cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 430200ae..3b02dedd 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -1,7 +1,6 @@ -#!/bin/sh -e +#!/bin/sh # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# 2020-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,22 +17,12 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# # All os variables are lower case # +# -rc_getvar() { - awk -F= -v varname="$2" ' - function unquote(s) { - if (s ~ /^".*"$/ || s ~ /^'\''.*'\''$/) - return substr(s, 2, length(s) - 2) - else - return s - } - $1 == varname { print unquote(substr($0, index($0, "=") + 1)) }' "$1" -} - -case $("${__explorer:?}/os") -in +case "$("$__explorer/os")" in amazon) cat /etc/system-release ;; @@ -41,9 +30,6 @@ in # empty, but well... cat /etc/arch-release ;; - checkpoint) - awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release - ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version @@ -57,8 +43,6 @@ in # sid versions don't have a number, so we decode by codename: case $(expr "$debian_version" : '\([a-z]\{1,\}\)/') in - trixie) echo 12.99 ;; - bookworm) echo 11.99 ;; bullseye) echo 10.99 ;; buster) echo 9.99 ;; stretch) echo 8.99 ;; @@ -66,7 +50,7 @@ in wheezy) echo 6.99 ;; squeeze) echo 5.99 ;; lenny) echo 4.99 ;; - *) echo 99.99 ;; + *) exit 1 esac ;; *) @@ -75,23 +59,7 @@ in esac ;; devuan) - devuan_version=$(cat /etc/devuan_version) - case ${devuan_version} - in - (*/ceres) - # ceres versions don't have a number, so we decode by codename: - case ${devuan_version} - in - (chimaera/ceres) echo 3.99 ;; - (beowulf/ceres) echo 2.99 ;; - (ascii/ceres) echo 1.99 ;; - (*) exit 1 - esac - ;; - (*) - echo "${devuan_version}" - ;; - esac + cat /etc/devuan_version ;; fedora) cat /etc/fedora-release @@ -100,20 +68,12 @@ in cat /etc/gentoo-release ;; macosx) - # NOTE: Legacy versions (< 10.3) do not support options - sw_vers | awk -F ':[ \t]+' '$1 == "ProductVersion" { print $2 }' + sw_vers -productVersion ;; freebsd) # Apparently uname -r is not a reliable way to get the patch level. # See: https://bugs.freebsd.org/bugzilla/show_bug.cgi?id=251743 - if command -v freebsd-version >/dev/null 2>&1 - then - # get userland version - freebsd-version -u - else - # fallback to kernel release for FreeBSD < 10.0 - uname -r - fi + freebsd-version ;; *bsd|solaris) uname -r @@ -138,20 +98,7 @@ in fi ;; ubuntu) - if command -v lsb_release >/dev/null 2>&1 - then - lsb_release -sr - elif test -r /usr/lib/os-release - then - # fallback to /usr/lib/os-release if lsb_release is not present (like - # on minimized Ubuntu installations) - rc_getvar /usr/lib/os-release VERSION_ID - elif test -r /etc/lsb-release - then - # extract DISTRIB_RELEASE= variable from /etc/lsb-release on old - # versions without /usr/lib/os-release. - rc_getvar /etc/lsb-release DISTRIB_RELEASE - fi + lsb_release -sr ;; alpine) cat /etc/alpine-release diff --git a/cdist/conf/type/__apt_backports/manifest b/cdist/conf/type/__apt_backports/manifest index 6fcd9212..bc47d8de 100755 --- a/cdist/conf/type/__apt_backports/manifest +++ b/cdist/conf/type/__apt_backports/manifest @@ -28,7 +28,6 @@ # lsb_release may not be given in all installations codename_os_release() { # shellcheck disable=SC1090 - # shellcheck disable=SC1091 . "$__global/explorer/os_release" printf "%s" "$VERSION_CODENAME" } diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index 8ab268c1..38f1bd3c 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,25 +27,18 @@ else keyid="$__object_id" fi -# From apt-key(8): -# Use of apt-key is deprecated, except for the use of apt-key del in -# maintainer scripts to remove existing keys from the main keyring. -# If such usage of apt-key is desired the additional installation of -# the GNU Privacy Guard suite (packaged in gnupg) is required. -if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then - if apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" - then echo present - else echo absent - fi - exit -fi - keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" -if [ -f "$keyfile" ] +if [ -d "$keydir" ] then - echo present - exit + if [ -f "$keyfile" ] + then echo present + else echo absent + fi +else + # fallback to deprecated apt-key + apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ + && echo present \ + || echo absent fi -echo absent diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index 17dc9bfc..0c96ff67 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -25,7 +25,11 @@ else fi state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" -method="$(cat "$__object/key_method")" + +if [ "$state_should" = "$state_is" ]; then + # nothing to do + exit 0 +fi keydir="$(cat "$__object/parameter/keydir")" keyfile="$keydir/$__object_id.gpg" @@ -33,18 +37,30 @@ keyfile="$keydir/$__object_id.gpg" case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - # Using __download or __file as key source - # Propagate messages if needed - if [ "${method}" = "uri" ] || [ "${method}" = "source" ]; then - if grep -Eq "^__(file|download)$keyfile" "$__messages_in"; then - echo "added '$keyid'" >> "$__messages_out" + + if [ -f "$__object/parameter/uri" ]; then + uri="$(cat "$__object/parameter/uri")" + + if [ -d "$keydir" ]; then + cat << EOF + +curl -s -L \\ + -o "$keyfile" \\ + "$uri" + +key="\$( cat "$keyfile" )" + +if echo "\$key" | grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' +then + echo "\$key" | gpg --dearmor > "$keyfile" +fi + +EOF + else + # fallback to deprecated apt-key + echo "curl -s -L '$uri' | apt-key add -" fi - exit 0 - elif [ "${state_is}" = "present" ]; then - exit 0 - fi - # Using key servers to fetch the key - if [ ! -f "$__object/parameter/use-deprecated-apt-key" ]; then + elif [ -d "$keydir" ]; then # we need to kill gpg after 30 seconds, because gpg # can get stuck if keyserver is not responding. # exporting env var and not exit 1, @@ -84,16 +100,13 @@ EOF echo "added '$keyid'" >> "$__messages_out" ;; absent) - # Removal for keys added from a keyserver without this flag - # is done in the manifest - if [ "$state_is" != "absent" ] && \ - [ -f "$__object/parameter/use-deprecated-apt-key" ]; then + if [ -f "$keyfile" ]; then + echo "rm '$keyfile'" + else # fallback to deprecated apt-key echo "apt-key del \"$keyid\"" - echo "removed '$keyid'" >> "$__messages_out" - # Propagate messages if needed - elif grep -Eq "^__file$keyfile" "$__messages_in"; then - echo "removed '$keyid'" >> "$__messages_out" fi + + echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index e35eaa0f..234bc715 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -10,14 +10,6 @@ DESCRIPTION ----------- Manages the list of keys used by apt to authenticate packages. -This is done by placing the requested key in a file named -``$__object_id.gpg`` in the ``keydir`` directory. - -This is supported by modern releases of Debian-based distributions. - -In order of preference, exactly one of: ``source``, ``uri`` or ``keyid`` -must be specified. - REQUIRED PARAMETERS ------------------- @@ -26,49 +18,21 @@ None. OPTIONAL PARAMETERS ------------------- -keydir - keyring directory, defaults to ``/etc/apt/trusted.pgp.d``, which is - enabled system-wide by default. - -source - path to a file containing the GPG key of the repository. - Using this is recommended as it ensures that the manifest/type manintainer - has validated the key. - If ``-``, the GPG key is read from the type's stdin. - state 'present' or 'absent'. Defaults to 'present' -uri - the URI from which to download the key. - It is highly recommended that you only use protocols with TLS like HTTPS. - This uses ``__download`` but does not use checksums, if you want to ensure - that the key doesn't change, you are better off downloading it and using - ``--source``. - - -DEPRECATED OPTIONAL PARAMETERS ------------------------------- keyid - the id of the key to download from the ``keyserver``. - This is to be used in absence of ``--source`` and ``--uri`` or together - with ``--use-deprecated-apt-key`` for key removal. - Defaults to ``$__object_id``. + the id of the key to add. Defaults to __object_id keyserver - the keyserver from which to fetch the key. - Defaults to ``pool.sks-keyservers.net``. + the keyserver from which to fetch the key. If omitted the default set + in ./parameter/default/keyserver is used. +keydir + key save location, defaults to ``/etc/apt/trusted.pgp.d`` -DEPRECATED BOOLEAN PARAMETERS ------------------------------ -use-deprecated-apt-key - ``apt-key(8)`` will last be available in Debian 11 and Ubuntu 22.04. - You can use this parameter to force usage of ``apt-key(8)``. - Please only use this parameter to *remove* keys from the keyring, - in order to prepare for removal of ``apt-key``. - Adding keys should be done without this parameter. - This parameter will be removed when Debian 11 stops being supported. +uri + the URI from which to download the key EXAMPLES @@ -76,39 +40,33 @@ EXAMPLES .. code-block:: sh - # add a key that has been verified by a type maintainer - __apt_key jitsi_meet_2021 \ - --source cdist-contrib/type/__jitsi_meet/files/apt_2021.gpg + # Add Ubuntu Archive Automatic Signing Key + __apt_key 437D05B5 + # Same thing + __apt_key 437D05B5 --state present + # Get rid of it + __apt_key 437D05B5 --state absent - # remove an old, deprecated or expired key - __apt_key jitsi_meet_2016 --state absent + # same thing with human readable name and explicit keyid + __apt_key UbuntuArchiveKey --keyid 437D05B5 - # Get rid of a key that might have been added to - # /etc/apt/trusted.gpg with apt-key - __apt_key 0x40976EAF437D05B5 --use-deprecated-apt-key --state absent + # same thing with other keyserver + __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com - # add a key that we define in-line - __apt_key jitsi_meet_2021 --source '-' < Ander Punnar -Evilham COPYING ------- -Copyright \(C) 2011-2021 Steven Armstrong, Ander Punnar and Evilham. You can +Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest index 889a764a..010357cd 100755 --- a/cdist/conf/type/__apt_key/manifest +++ b/cdist/conf/type/__apt_key/manifest @@ -2,105 +2,7 @@ __package gnupg -state_should="$(cat "${__object}/parameter/state")" - -incompatible_args() -{ - cat >> /dev/stderr <<-EOF - This type does not support --${1} and --${method} simultaneously. - EOF - exit 1 -} - -if [ -f "${__object}/parameter/source" ]; then - method="source" - src="$(cat "${__object}/parameter/source")" - if [ "${src}" = "-" ]; then - src="${__object}/stdin" - fi -fi -if [ -f "${__object}/parameter/uri" ]; then - if [ -n "${method}" ]; then - incompatible_args uri - fi - method="uri" - src="$(cat "${__object}/parameter/uri")" -fi -if [ -f "${__object}/parameter/keyid" ]; then - if [ -n "${method}" ]; then - incompatible_args keyid - fi - method="keyid" -fi -# Keep old default -if [ -z "${method}" ]; then - method="keyid" -fi -# Save this for later in gencode-remote -echo "${method}" > "${__object}/key_method" - -# Required remotely (most likely already installed) -__package dirmngr -# We need this in case a key has to be dearmor'd -__package gnupg -export require="__package/gnupg" - -if [ -f "${__object}/parameter/use-deprecated-apt-key" ]; then - # This is required if apt-key(8) is to be used - if [ "${method}" = "source" ] || [ "${method}" = "uri" ]; then - incompatible_args use-deprecated-apt-key - fi -else - if [ "${state_should}" = "absent" ] && \ - [ -f "${__object}/parameter/keyid" ]; then - cat >> /dev/stderr < - - -COPYING -------- -Copyright \(C) 2021 Daniel Fancsali. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority deleted file mode 100644 index 1b79f38e..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/priority +++ /dev/null @@ -1 +0,0 @@ -500 diff --git a/cdist/conf/type/__apt_pin/parameter/default/state b/cdist/conf/type/__apt_pin/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional deleted file mode 100644 index 847e703d..00000000 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ /dev/null @@ -1,3 +0,0 @@ -state -package -priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required deleted file mode 100644 index c8572d92..00000000 --- a/cdist/conf/type/__apt_pin/parameter/required +++ /dev/null @@ -1 +0,0 @@ -distribution diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository new file mode 100755 index 00000000..3eb7d491 --- /dev/null +++ b/cdist/conf/type/__apt_ppa/files/remove-apt-repository @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Remove the given apt repository. +# +# Exit with: +# 0: if it worked +# 1: if not +# 2: on other error + +import os +import sys +from aptsources import distro, sourceslist +from softwareproperties import ppa +from softwareproperties.SoftwareProperties import SoftwareProperties + + +def remove_if_empty(file_name): + with open(file_name, 'r') as f: + if f.read().strip(): + return + os.unlink(file_name) + +def remove_repository(repository): + #print 'repository:', repository + codename = distro.get_distro().codename + #print 'codename:', codename + (line, file) = ppa.expand_ppa_line(repository.strip(), codename) + #print 'line:', line + #print 'file:', file + deb_source_entry = sourceslist.SourceEntry(line, file) + src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) + + try: + sp = SoftwareProperties() + sp.remove_source(deb_source_entry) + try: + # If there's a deb-src entry, remove that too + sp.remove_source(src_source_entry) + except: + pass + remove_if_empty(file) + return True + except ValueError: + print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line + return False + +if __name__ == '__main__': + if (len(sys.argv) != 2): + print >> sys.stderr, 'Error: need a repository as argument' + sys.exit(2) + repository = sys.argv[1] + if remove_repository(repository): + sys.exit(0) + else: + sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index e41341b8..84ebebfe 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,9 +29,9 @@ fi case "$state_should" in present) - echo "add-apt-repository -y '$name'" + echo "add-apt-repository '$name'" ;; absent) - echo "add-apt-repository -r -y '$name'" + echo "remove-apt-repository '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index 57e85442..c6f4e876 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,4 +20,9 @@ __package software-properties-common +require="__package/software-properties-common" \ + __file /usr/local/bin/remove-apt-repository \ + --source "$__type/files/remove-apt-repository" \ + --mode 0755 + require="$__object_name" __apt_update_index diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index a28bb45f..d4420e96 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,14 +2,13 @@ set -u entry="$uri $distribution $component" - cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${options} $entry +deb ${forcedarch} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 973b0f6c..1e8592c6 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,21 +22,7 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - -# run 'apt-get update' only if something changed with our sources.list file -# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" + printf 'apt-get update || apt-get update\n' fi diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d317a135..d1acb388 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,9 +23,6 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) -signed-by - provide a GPG key fingerprint or keyring path for signature checks - state 'present' or 'absent', defaults to 'present' @@ -59,11 +56,6 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present - __apt_source goaccess \ - --uri http://deb.goaccess.io/ \ - --component main \ - --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D - AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index cdb526d3..35f15909 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -31,15 +31,9 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - options="arch=$(cat "$__object/parameter/arch")" -fi - -if [ -f "$__object/parameter/signed-by" ]; then - options="$options signed-by=$(cat "$__object/parameter/signed-by")" -fi - -if [ "$options" ]; then - options="[$options]" + forcedarch="[arch=$(cat "$__object/parameter/arch")]" +else + forcedarch="" fi # export variables for use in template @@ -47,7 +41,7 @@ export name export uri export distribution export component -export options +export forcedarch # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 0b5470a1..87537335 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,5 +1,4 @@ state distribution component -arch -signed-by +arch \ No newline at end of file diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 2d7f9030..70b59710 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,23 +18,9 @@ # along with cdist. If not, see . # - -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists -# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get $apt_opts update || apt-get $apt_opts update + apt-get update || apt-get update fi DONE diff --git a/cdist/conf/type/__debconf_set_selections/explorer/state b/cdist/conf/type/__debconf_set_selections/explorer/state deleted file mode 100644 index f8a3f6c8..00000000 --- a/cdist/conf/type/__debconf_set_selections/explorer/state +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh -e -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# Determine current debconf selections' state. -# Prints one of: -# present: all selections are already set as they should. -# different: one or more of the selections have a different value. -# absent: one or more of the selections are not (currently) defined. -# - -test -x /usr/bin/perl || { - # cannot find perl (no perl ~ no debconf) - echo 'absent' - exit 0 -} - -linesfile="${__object:?}/parameter/line" -test -s "${linesfile}" || { - if test -s "${__object:?}/parameter/file" - then - echo absent - else - echo present - fi - exit 0 -} - -# assert __type_explorer is set (because it is used by the Perl script) -: "${__type_explorer:?}" - -/usr/bin/perl -- - "${linesfile}" <<'EOF' -use strict; -use warnings "all"; - -use Fcntl qw(:DEFAULT :flock); - -use Debconf::Db; -use Debconf::Question; - -# Extract @known... arrays from debconf-set-selections -# These values are required to distinguish flags and values in the given lines. -# DC: I couldn't think of a more ugly solution to the problem… -my @knownflags; -my @knowntypes; -my $debconf_set_selections = '/usr/bin/debconf-set-selections'; -if (-e $debconf_set_selections) { - my $sed_known = 's/^my \(@known\(flags\|types\) = qw([a-z ]*);\).*$/\1/p'; - eval `sed -n '$sed_known' '$debconf_set_selections'`; -} - -sub mungeline ($) { - my $line = shift; - chomp $line; - $line =~ s/\r$//; - return $line; -} - -sub fatal { printf STDERR @_; exit 1; } - -my $state = 'present'; - -sub state { - my $new = shift; - if ($state eq 'present' - or ($state eq 'different' and $new eq 'absent')) { - $state = $new; - } -} - - -# Load Debconf DB but manually lock on the state explorer script, -# because Debconf aborts immediately if executed concurrently. -# This is not really an ideal solution because the Debconf DB could be locked by -# another process (e.g. apt-get), but no way to achieve this could be found. -# If you know how to, please provide a patch. -my $lockfile = "%ENV{'__type_explorer'}/state"; -if (open my $lock_fh, '+<', $lockfile) { - flock $lock_fh, LOCK_EX or die "Cannot lock $lockfile"; -} -{ - Debconf::Db->load(readonly => 'true'); -} - - -while (<>) { - # Read and process lines (taken from debconf-set-selections) - $_ = mungeline($_); - while (/\\$/ && ! eof) { - s/\\$//; - $_ .= mungeline(<>); - } - next if /^\s*$/ || /^\s*\#/; - - my ($owner, $label, $type, $content) = /^\s*(\S+)\s+(\S+)\s+(\S+)(?:\s(.*))?/ - or fatal "invalid line: %s\n", $_; - $content = '' unless defined $content; - - - # Compare is and should state - my $q = Debconf::Question->get($label); - - unless (defined $q) { - # probably a preseed - state 'absent'; - next; - } - - if (grep { $_ eq $q->type } @knownflags) { - # This line wants to set a flag, presumably. - if ($q->flag($q->type) ne $content) { - state 'different'; - } - } else { - # Otherwise, it's probably a value… - if ($q->value ne $content) { - state 'different'; - } - - unless (grep { $_ eq $owner } (split /, /, $q->owners)) { - state 'different'; - } - } -} - -printf "%s\n", $state; -EOF diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index 9ba28f09..e99aef40 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -18,37 +17,16 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# +# Setup selections +# -if test -f "${__object:?}/parameter/line" -then - filename="${__object:?}/parameter/line" -elif test -s "${__object:?}/parameter/file" -then - filename=$(cat "${__object:?}/parameter/file") - if test "${filename}" = '-' - then - filename="${__object:?}/stdin" - fi -else - printf 'Neither --line nor --file set.\n' >&2 - exit 1 +filename="$(cat "$__object/parameter/file")" + +if [ "$filename" = "-" ]; then + filename="$__object/stdin" fi -# setting no lines makes no sense -test -s "${filename}" || exit 0 - -state_is=$(cat "${__object:?}/explorer/state") - -if test "${state_is}" != 'present' -then - cat <<-CODE - debconf-set-selections <<'EOF' - $(cat "${filename}") - EOF - CODE - - awk ' - { - printf "set %s %s %s %s\n", $1, $2, $3, $4 - }' "${filename}" >>"${__messages_out:?}" -fi +echo "debconf-set-selections << __file-eof" +cat "$filename" +echo "__file-eof" diff --git a/cdist/conf/type/__debconf_set_selections/man.rst b/cdist/conf/type/__debconf_set_selections/man.rst index fd0040ae..58c25b81 100644 --- a/cdist/conf/type/__debconf_set_selections/man.rst +++ b/cdist/conf/type/__debconf_set_selections/man.rst @@ -8,33 +8,15 @@ cdist-type__debconf_set_selections - Setup debconf selections DESCRIPTION ----------- -On Debian and alike systems :strong:`debconf-set-selections`\ (1) can be used +On Debian and alike systems debconf-set-selections(1) can be used to setup configuration parameters. REQUIRED PARAMETERS ------------------- -cf. ``--line``. - - -OPTIONAL PARAMETERS -------------------- file - Use the given filename as input for :strong:`debconf-set-selections`\ (1) - If filename is ``-``, read from stdin. - - **This parameter is deprecated, because it doesn't work with state detection.** -line - A line in :strong:`debconf-set-selections`\ (1) compatible format. - This parameter can be used multiple times to set multiple options. - - (This parameter is actually required, but marked optional because the - deprecated ``--file`` is still accepted.) - - -BOOLEAN PARAMETERS ------------------- -None. + Use the given filename as input for debconf-set-selections(1) + If filename is "-", read from stdin. EXAMPLES @@ -42,29 +24,30 @@ EXAMPLES .. code-block:: sh - # Setup gitolite's gituser - __debconf_set_selections nslcd --line 'gitolite gitolite/gituser string git' + # Setup configuration for nslcd + __debconf_set_selections nslcd --file /path/to/file - # Setup configuration for nslcd from a file. - # NB: Multiple lines can be passed to --line, although this can be considered a hack. - __debconf_set_selections nslcd --line "$(cat "${__files:?}/preseed/nslcd.debconf")" + # Setup configuration for nslcd from another type + __debconf_set_selections nslcd --file "$__type/files/preseed/nslcd" + + __debconf_set_selections nslcd --file - << eof + gitolite gitolite/gituser string git + eof SEE ALSO -------- -- :strong:`cdist-type__update_alternatives`\ (7) -- :strong:`debconf-set-selections`\ (1) +:strong:`debconf-set-selections`\ (1), :strong:`cdist-type__update_alternatives`\ (7) AUTHORS ------- -| Nico Schottelius -| Dennis Camera +Nico Schottelius COPYING ------- -Copyright \(C) 2011-2014 Nico Schottelius, 2021 Dennis Camera. -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. +Copyright \(C) 2011-2014 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__debconf_set_selections/nonparallel b/cdist/conf/type/__debconf_set_selections/nonparallel deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file deleted file mode 100644 index 09db545a..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file +++ /dev/null @@ -1 +0,0 @@ -'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple b/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple deleted file mode 100644 index a999a0c2..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/optional_multiple +++ /dev/null @@ -1 +0,0 @@ -line diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional b/cdist/conf/type/__debconf_set_selections/parameter/required similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/optional rename to cdist/conf/type/__debconf_set_selections/parameter/required diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index c8f36712..ba7621a1 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,12 +37,6 @@ state source forwarded to :strong:`__file` type -file - forwarded to :strong:`__file` type - This can be used if multiple users need to have a dotfile updated, - which will result in duplicate object id errors. When using the - file parameter the object id can be some unique value. - MESSAGES -------- @@ -67,15 +61,6 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" - # install .vimrc for root and some users - for user in root userx usery userz; do - __dot_file "${user}_dot_vimrc" \ - --user $user \ - --file .vimrc \ - --state exists \ - --source "$__files/$user/.vimrc" - done - SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index a38ed943..02dadf05 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,19 +20,13 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" -if [ -f "${__object}/parameter/file" ]; then - file="$(cat "${__object}/parameter/file")" -else - file="${__object_id}" -fi - # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${file} +subpath=${__object_id} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -70,4 +64,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" diff --git a/cdist/conf/type/__download/explorer/remote_cmd b/cdist/conf/type/__download/explorer/remote_cmd new file mode 100755 index 00000000..e3e35b45 --- /dev/null +++ b/cdist/conf/type/__download/explorer/remote_cmd @@ -0,0 +1,19 @@ +#!/bin/sh -e + +if [ -f "$__object/parameter/cmd-get" ] +then + cmd="$( cat "$__object/parameter/cmd-get" )" + +elif command -v curl > /dev/null +then + cmd="curl -L -o - '%s'" + +elif command -v fetch > /dev/null +then + cmd="fetch -o - '%s'" + +else + cmd="wget -O - '%s'" +fi + +echo "$cmd" diff --git a/cdist/conf/type/__download/explorer/remote_cmd_get b/cdist/conf/type/__download/explorer/remote_cmd_get deleted file mode 100755 index 9f1cd59c..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd_get +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/cmd-get" ] -then - cat "$__object/parameter/cmd-get" -elif - command -v curl > /dev/null -then - echo "curl -sSL -o - '%s'" -elif - command -v fetch > /dev/null -then - echo "fetch -o - '%s'" -else - echo "wget -O - '%s'" -fi diff --git a/cdist/conf/type/__download/explorer/remote_cmd_sum b/cdist/conf/type/__download/explorer/remote_cmd_sum deleted file mode 100755 index 84df663c..00000000 --- a/cdist/conf/type/__download/explorer/remote_cmd_sum +++ /dev/null @@ -1,82 +0,0 @@ -#!/bin/sh -e - -if [ ! -f "$__object/parameter/sum" ] -then - exit 0 -fi - -if [ -f "$__object/parameter/cmd-sum" ] -then - cat "$__object/parameter/cmd-sum" - exit 0 -fi - -sum_should="$( cat "$__object/parameter/sum" )" - -if echo "$sum_should" | grep -Fq ':' -then - sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" -else - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_hash='cksum' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' - then - sum_hash='md5' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' - then - sum_hash='sha1' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' - then - sum_hash='sha256' - else - echo 'hash format detection failed' >&2 - exit 1 - fi -fi - -os="$( "$__explorer/os" )" - -case "$sum_hash" in - cksum) - echo "cksum %s | awk '{print \$1\" \"\$2}'" - ;; - md5) - case "$os" in - freebsd) - echo "md5 -q %s" - ;; - *) - echo "md5sum %s | awk '{print \$1}'" - ;; - esac - ;; - sha1) - case "$os" in - freebsd) - echo "sha1 -q %s" - ;; - *) - echo "sha1sum %s | awk '{print \$1}'" - ;; - esac - ;; - sha256) - case "$os" in - freebsd) - echo "sha256 -q %s" - ;; - *) - echo "sha256sum %s | awk '{print \$1}'" - ;; - esac - ;; - *) - # we arrive here only if --sum is given with unknown format prefix - echo "unknown hash format: $sum_hash" >&2 - exit 1 - ;; -esac diff --git a/cdist/conf/type/__download/explorer/state b/cdist/conf/type/__download/explorer/state index 8c5d5ce1..00362545 100755 --- a/cdist/conf/type/__download/explorer/state +++ b/cdist/conf/type/__download/explorer/state @@ -1,11 +1,6 @@ #!/bin/sh -e -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi +dst="/$__object_id" if [ ! -f "$dst" ] then @@ -13,27 +8,59 @@ then exit 0 fi -if [ ! -f "$__object/parameter/sum" ] -then - echo 'present' - exit 0 -fi - sum_should="$( cat "$__object/parameter/sum" )" -if echo "$sum_should" | grep -Fq ':' +if [ -f "$__object/parameter/cmd-sum" ] then - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" + # shellcheck disable=SC2059 + sum_is="$( eval "$( printf \ + "$( cat "$__object/parameter/cmd-sum" )" \ + "$dst" )" )" +else + os="$( "$__explorer/os" )" + + if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' + then + sum_is="$( cksum "$dst" | awk '{print $1" "$2}' )" + + elif echo "$sum_should" | grep -Eiq '^md5:[a-f0-9]{32}$' + then + case "$os" in + freebsd) + sum_is="md5:$( md5 -q "$dst" )" + ;; + *) + sum_is="md5:$( md5sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha1:[a-f0-9]{40}$' + then + case "$os" in + freebsd) + sum_is="sha1:$( sha1 -q "$dst" )" + ;; + *) + sum_is="sha1:$( sha1sum "$dst" | awk '{print $1}' )" + ;; + esac + + elif echo "$sum_should" | grep -Eiq '^sha256:[a-f0-9]{64}$' + then + case "$os" in + freebsd) + sum_is="sha256:$( sha256 -q "$dst" )" + ;; + *) + sum_is="sha256:$( sha256sum "$dst" | awk '{print $1}' )" + ;; + esac + fi fi -sum_cmd="$( "$__type_explorer/remote_cmd_sum" )" - -# shellcheck disable=SC2059 -sum_is="$( eval "$( printf "$sum_cmd" "'$dst'" )" )" - if [ -z "$sum_is" ] then - echo 'existing destination checksum failed' >&2 + echo 'no checksum from target' >&2 exit 1 fi diff --git a/cdist/conf/type/__download/gencode-local b/cdist/conf/type/__download/gencode-local index d1b0d0d5..571d2c3c 100755 --- a/cdist/conf/type/__download/gencode-local +++ b/cdist/conf/type/__download/gencode-local @@ -11,133 +11,34 @@ fi url="$( cat "$__object/parameter/url" )" -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi +tmp="$( mktemp )" + +dst="/$__object_id" if [ -f "$__object/parameter/cmd-get" ] then cmd="$( cat "$__object/parameter/cmd-get" )" +elif command -v wget > /dev/null +then + cmd="wget -O - '%s'" + elif command -v curl > /dev/null then - cmd="curl -sSL -o - '%s'" + cmd="curl -L -o - '%s'" elif command -v fetch > /dev/null then cmd="fetch -o - '%s'" -elif command -v wget > /dev/null -then - cmd="wget -O - '%s'" - else - echo 'local download failed, no usable utility' >&2 + echo 'no usable locally installed utility for downloading' >&2 exit 1 fi -echo "download_tmp=\"\$( mktemp )\"" - -# shellcheck disable=SC2059 -printf "$cmd > \"\$download_tmp\"\n" "$url" - -if [ -f "$__object/parameter/sum" ] -then - sum_should="$( cat "$__object/parameter/sum" )" - - if [ -f "$__object/parameter/cmd-sum" ] - then - local_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" - else - if echo "$sum_should" | grep -Fq ':' - then - sum_hash="$( echo "$sum_should" | cut -d : -f 1 )" - - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" - else - if echo "$sum_should" | grep -Eq '^[0-9]+\s[0-9]+$' - then - sum_hash='cksum' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{32}$' - then - sum_hash='md5' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{40}$' - then - sum_hash='sha1' - elif - echo "$sum_should" | grep -Eiq '^[a-f0-9]{64}$' - then - sum_hash='sha256' - else - echo 'hash format detection failed' >&2 - exit 1 - fi - fi - - case "$sum_hash" in - cksum) - local_cmd_sum="cksum %s | awk '{print \$1\" \"\$2}'" - ;; - md5) - if command -v md5 > /dev/null - then - local_cmd_sum="md5 -q %s" - elif - command -v md5sum > /dev/null - then - local_cmd_sum="md5sum %s | awk '{print \$1}'" - fi - ;; - sha1) - if command -v sha1 > /dev/null - then - local_cmd_sum="sha1 -q %s" - elif - command -v sha1sum > /dev/null - then - local_cmd_sum="sha1sum %s | awk '{print \$1}'" - fi - ;; - sha256) - if command -v sha256 > /dev/null - then - local_cmd_sum="sha256 -q %s" - elif - command -v sha256sum > /dev/null - then - local_cmd_sum="sha256sum %s | awk '{print \$1}'" - fi - ;; - *) - # we arrive here only if --sum is given with unknown format prefix - echo "unknown hash format: $sum_hash" >&2 - exit 1 - ;; - esac - - if [ -z "$local_cmd_sum" ] - then - echo 'local checksum verification failed, no usable utility' >&2 - exit 1 - fi - fi - - # shellcheck disable=SC2059 - echo "sum_is=\"\$( $( printf "$local_cmd_sum" "\"\$download_tmp\"" ) )\"" - - echo "if [ \"\$sum_is\" != '$sum_should' ]; then" - - echo "echo 'local download checksum mismatch' >&2" - - echo "rm -f \"\$download_tmp\"" - - echo 'exit 1; fi' -fi +printf "$cmd > %s\n" \ + "$url" \ + "$tmp" if echo "$__target_host" | grep -Eq '^[0-9a-fA-F:]+$' then @@ -146,10 +47,12 @@ else target_host="$__target_host" fi -# shellcheck disable=SC2016 -printf '%s "$download_tmp" %s:%s\n' \ +printf '%s %s %s:%s\n' \ "$__remote_copy" \ + "$tmp" \ "$target_host" \ "$dst" -echo "rm -f \"\$download_tmp\"" +echo "rm -f '$tmp'" + +echo 'downloaded' > "$__messages_out" diff --git a/cdist/conf/type/__download/gencode-remote b/cdist/conf/type/__download/gencode-remote index e49bcec3..029a0801 100755 --- a/cdist/conf/type/__download/gencode-remote +++ b/cdist/conf/type/__download/gencode-remote @@ -6,51 +6,17 @@ state_is="$( cat "$__object/explorer/state" )" if [ "$download" = 'remote' ] && [ "$state_is" != 'present' ] then - cmd_get="$( cat "$__object/explorer/remote_cmd_get" )" + cmd="$( cat "$__object/explorer/remote_cmd" )" url="$( cat "$__object/parameter/url" )" - if [ -f "$__object/parameter/destination" ] - then - dst="$( cat "$__object/parameter/destination" )" - else - dst="/$__object_id" - fi + dst="/$__object_id" - echo "download_tmp=\"\$( mktemp )\"" + printf "$cmd > %s\n" \ + "$url" \ + "$dst" - # shellcheck disable=SC2059 - printf "$cmd_get > \"\$download_tmp\"\n" "$url" - - if [ -f "$__object/parameter/sum" ] - then - sum_should="$( cat "$__object/parameter/sum" )" - - if [ -f "$__object/parameter/cmd-sum" ] - then - remote_cmd_sum="$( cat "$__object/parameter/cmd-sum" )" - else - remote_cmd_sum="$( cat "$__object/explorer/remote_cmd_sum" )" - - if echo "$sum_should" | grep -Fq ':' - then - sum_should="$( echo "$sum_should" | cut -d : -f 2 )" - fi - fi - - # shellcheck disable=SC2059 - echo "sum_is=\"\$( $( printf "$remote_cmd_sum" "\"\$download_tmp\"" ) )\"" - - echo "if [ \"\$sum_is\" != '$sum_should' ]; then" - - echo "echo 'remote download checksum mismatch' >&2" - - echo "rm -f \"\$download_tmp\"" - - echo 'exit 1; fi' - fi - - echo "mv \"\$download_tmp\" '$dst'" + echo 'downloaded' > "$__messages_out" fi if [ -f "$__object/parameter/onchange" ] && [ "$state_is" != "present" ] diff --git a/cdist/conf/type/__download/man.rst b/cdist/conf/type/__download/man.rst index c16510a9..54503470 100644 --- a/cdist/conf/type/__download/man.rst +++ b/cdist/conf/type/__download/man.rst @@ -8,7 +8,10 @@ cdist-type__download - Download a file DESCRIPTION ----------- -By default type will try to use ``curl``, ``fetch`` or ``wget``. +Destination (``$__object_id``) in target host must be persistent storage +in order to calculate checksum and decide if file must be (re-)downloaded. + +By default type will try to use ``wget``, ``curl`` or ``fetch``. If download happens in target (see ``--download``) then type will fallback to (and install) ``wget``. @@ -16,40 +19,23 @@ If download happens in local machine, then environment variables like ``{http,https,ftp}_proxy`` etc can be used on cdist execution (``http_proxy=foo cdist config ...``). -To change downloaded file's owner, group or permissions, use ``require='__download/path/to/file' __file ...``. - REQUIRED PARAMETERS ------------------- url File's URL. +sum + Checksum of file going to be downloaded. + By default output of ``cksum`` without filename is expected. + Other hash formats supported with prefixes: ``md5:``, ``sha1:`` and ``sha256:``. + OPTIONAL PARAMETERS ------------------- -destination - Downloaded file's destination in target. If unset, ``$__object_id`` is used. - -sum - Supported formats: ``cksum`` output without file name, MD5, SHA1 and SHA256. - - Type tries to detect hash format with regexes, but prefixes - ``cksum:``, ``md5:``, ``sha1:`` and ``sha256:`` are also supported. - - Checksum have two purposes - state check and post-download verification. - In state check, if destination checksum mismatches, then content of URL - will be downloaded to temporary file. If downloaded temporary file's - checksum matches, then it will be moved to destination (overwritten). - - For local downloads it is expected that usable utilities for checksum - calculation exist in the system. - download - If ``local`` (default), then file is downloaded to local storage and copied - to target host. If ``remote``, then download happens in target. - - For local downloads it is expected that usable utilities for downloading - exist in the system. Type will try to use ``curl``, ``fetch`` or ``wget``. + If ``local`` (default), then download file to local storage and copy + it to target host. If ``remote``, then download happens in target. cmd-get Command used for downloading. @@ -79,7 +65,7 @@ EXAMPLES require='__directory/opt/cpma' \ __download /opt/cpma/cnq3.zip \ --url https://cdn.playmorepromode.com/files/cnq3/cnq3-1.51.zip \ - --sum 46da3021ca9eace277115ec9106c5b46 + --sum md5:46da3021ca9eace277115ec9106c5b46 require='__download/opt/cpma/cnq3.zip' \ __unpack /opt/cpma/cnq3.zip \ @@ -95,7 +81,7 @@ Ander Punnar COPYING ------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it +Copyright \(C) 2020 Ander Punnar. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__download/manifest b/cdist/conf/type/__download/manifest index 3d4c498b..7ec8d86d 100755 --- a/cdist/conf/type/__download/manifest +++ b/cdist/conf/type/__download/manifest @@ -1,6 +1,6 @@ #!/bin/sh -e -if grep -Eq '^wget' "$__object/explorer/remote_cmd_get" +if grep -Eq '^wget' "$__object/explorer/remote_cmd" then __package wget fi diff --git a/cdist/conf/type/__download/parameter/optional b/cdist/conf/type/__download/parameter/optional index e809ef78..838e2fbf 100644 --- a/cdist/conf/type/__download/parameter/optional +++ b/cdist/conf/type/__download/parameter/optional @@ -1,6 +1,4 @@ cmd-get cmd-sum -destination download onchange -sum diff --git a/cdist/conf/type/__download/parameter/required b/cdist/conf/type/__download/parameter/required index 96cdd3b9..6ea4c38f 100644 --- a/cdist/conf/type/__download/parameter/required +++ b/cdist/conf/type/__download/parameter/required @@ -1 +1,2 @@ url +sum diff --git a/cdist/conf/type/__fail2ban/gencode-remote b/cdist/conf/type/__fail2ban/gencode-remote new file mode 100755 index 00000000..900d054e --- /dev/null +++ b/cdist/conf/type/__fail2ban/gencode-remote @@ -0,0 +1,24 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +if [ -f "$__object/parameter/path-override" ]; then + echo "fail2ban-client reload" +fi diff --git a/cdist/conf/type/__fail2ban/man.rst b/cdist/conf/type/__fail2ban/man.rst new file mode 100644 index 00000000..81743d15 --- /dev/null +++ b/cdist/conf/type/__fail2ban/man.rst @@ -0,0 +1,53 @@ +cdist-type__fail2ban(7) +======================= + +NAME +---- +cdist-type__fail2ban - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __fail2ban + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__fail2ban/manifest similarity index 50% rename from cdist/conf/type/__apt_pin/manifest rename to cdist/conf/type/__fail2ban/manifest index 983b2b42..a557eafb 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__fail2ban/manifest @@ -19,50 +19,27 @@ # -name="$__object_id" - os=$(cat "$__global/explorer/os") -state="$(cat "$__object/parameter/state")" - -if [ -f "$__object/parameter/package" ]; then - package="$(cat "$__object/parameter/package")" -else - package=$name -fi - -distribution="$(cat "$__object/parameter/distribution")" -priority="$(cat "$__object/parameter/priority")" - case "$os" in debian|ubuntu|devuan) + : ;; *) - printf "This type is specific to Debian and it's derivatives" >&2 + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 exit 1 ;; esac -case $distribution in - stable|testing|unstable|experimental) - pin="release a=$distribution" - ;; - *) - pin="release n=$distribution" - ;; -esac +__package_apt fail2ban -__file "/etc/apt/preferences.d/$name" \ - --owner root --group root --mode 0644 \ - --state "$state" \ - --source - << EOF -# Created by cdist ${__type##*/} -# Do not change. Changes will be overwritten. -# +export require="__package_apt/fail2ban" -# $name -Package: $package -Pin: $pin -Pin-Priority: $priority -EOF +if [ -f "$__object/parameter/path-override" ]; then + ( + echo [DEFAULT] + cat "$__object/parameter/path-override" + ) | __file /etc/fail2ban/paths-overrides.local --mode 644 --source - +fi diff --git a/cdist/conf/type/__fail2ban/parameter/optional_multiple b/cdist/conf/type/__fail2ban/parameter/optional_multiple new file mode 100644 index 00000000..660fbeff --- /dev/null +++ b/cdist/conf/type/__fail2ban/parameter/optional_multiple @@ -0,0 +1 @@ +path-override diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__fail2ban/singleton similarity index 100% rename from cdist/conf/type/__haproxy_dualstack/singleton rename to cdist/conf/type/__fail2ban/singleton diff --git a/cdist/conf/type/__debconf_set_selections/manifest b/cdist/conf/type/__fail2ban_jail/gencode-remote similarity index 89% rename from cdist/conf/type/__debconf_set_selections/manifest rename to cdist/conf/type/__fail2ban_jail/gencode-remote index 0f4fb2e2..08d873ae 100755 --- a/cdist/conf/type/__debconf_set_selections/manifest +++ b/cdist/conf/type/__fail2ban_jail/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh -e # -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2021 Daniel Fancsali (fancsali@gmail.com) # # This file is part of cdist. # @@ -18,4 +18,5 @@ # along with cdist. If not, see . # -__package_apt debconf + +echo "fail2ban-client reload" diff --git a/cdist/conf/type/__fail2ban_jail/man.rst b/cdist/conf/type/__fail2ban_jail/man.rst new file mode 100644 index 00000000..27276990 --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/man.rst @@ -0,0 +1,53 @@ +cdist-type__fail2ban_jail(7) +============================ + +NAME +---- +cdist-type__fail2ban_jail - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + __fail2ban_jail + + +SEE ALSO +-------- +:strong:`TODO`\ (7) + + +AUTHORS +------- +Daniel Fancsali + + +COPYING +------- +Copyright \(C) 2021 Daniel Fancsali. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__fail2ban_jail/manifest b/cdist/conf/type/__fail2ban_jail/manifest new file mode 100755 index 00000000..47b8a2d6 --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/manifest @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2021 Daniel Fancsali (fancsali@gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + debian|ubuntu|devuan) + : + ;; + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "$os" "${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac + +export require="__fail2ban" + +__file /etc/fail2ban/jail.local --mode 644 + +export require="__file/etc/fail2ban/jail.local" +( + echo "[$__object_id]" + echo "enabled = true" + if [ -f "$__object/parameter/override" ]; then + cat "$__object/parameter/override" + fi +) | __block "/etc/fail2ban/jail.local:$__object_id" --file /etc/fail2ban/jail.local --text - diff --git a/cdist/conf/type/__apt_pin/nonparallel b/cdist/conf/type/__fail2ban_jail/nonparallel similarity index 100% rename from cdist/conf/type/__apt_pin/nonparallel rename to cdist/conf/type/__fail2ban_jail/nonparallel diff --git a/cdist/conf/type/__fail2ban_jail/parameter/optional_multiple b/cdist/conf/type/__fail2ban_jail/parameter/optional_multiple new file mode 100644 index 00000000..9ab91d29 --- /dev/null +++ b/cdist/conf/type/__fail2ban_jail/parameter/optional_multiple @@ -0,0 +1 @@ +override diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 5a303308..231b6927 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -72,7 +72,6 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 - echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -89,39 +88,27 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" - if [ "$create_file" ]; then - # When creating an empty file we create it locally and then - # upload it so that permissions can be set before moving the file - # into place. - source="$__object/files/empty" - touch "$source" - fi - # upload file to temp location - upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" - # Yes, we are aware that this is a race condition. - # However: - # a) cdist usually writes to directories that are not user writable - # (probably > 99.9%) - # b) if they are user owned, the user / attacker always wins - # (probably < 0.1%) - # c) the only case which we could improve are tmp directories and we - # don't think managing tmp directories with cdist is a typical case - # ("the rest %)" - - # Tell gencode-remote to where we uploaded the file so it can move - # it to its final destination. - echo "$upload_destination" > "$__object/files/upload-destination" - - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi + tempfile_template="${destination}.cdist.XXXXXXXXXX" cat << DONE -$__remote_copy "$source" "${my_target_host}:${upload_destination}" +destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +DONE + if [ "$upload_file" ]; then + echo upload >> "$__messages_out" + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE +$__remote_copy "$source" "${my_target_host}:\$destination_upload" +DONE + fi +# move uploaded file into place +cat << DONE +$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index 1a9ff69c..f7a528fd 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,13 +62,6 @@ set_mode() { case "$state_should" in present|exists) - if [ -f "$__object/files/upload-destination" ]; then - final_destination="$destination" - # We change the 'global' $destination variable here so we can - # change attributes of the new/uploaded file before moving it - # to it's final destination. - destination="$(cat "$__object/files/upload-destination")" - fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -88,11 +81,6 @@ case "$state_should" in fi fi done - if [ -f "$__object/files/upload-destination" ]; then - # move uploaded file into place - printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv "%s" "%s"\n' "$destination" "$final_destination" - fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 diff --git a/cdist/conf/type/__filesystem/explorer/lsblk b/cdist/conf/type/__filesystem/explorer/lsblk index d376c09f..9be3c575 100644 --- a/cdist/conf/type/__filesystem/explorer/lsblk +++ b/cdist/conf/type/__filesystem/explorer/lsblk @@ -27,7 +27,7 @@ else fi case "$os" in - alpine|centos|fedora|gentoo|redhat|suse|ubuntu) + alpine|centos|fedora|redhat|suse|gentoo) if [ ! -x "$(command -v lsblk)" ]; then echo "lsblk is required for __filesystem type" >&2 exit 1 diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index ab4396b1..3ddf9656 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -1,24 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}/.git" +destination="/$__object_id/.git" -# shellcheck disable=SC2012 -group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((group_gid + 1)) -ge 0 -then - group_should=$(cat "${__object:?}/parameter/group") - - if expr "${group_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${group_gid}" - else - if command -v getent > /dev/null - then - getent group "${group_gid}" | cut -d : -f 1 - else - awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group - fi - fi -fi +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 4a4d0d13..4c3cd431 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -1,19 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}/.git" +destination="/$__object_id/.git" -# shellcheck disable=SC2012 -owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((owner_uid + 1)) -ge 0 -then - owner_should=$(cat "${__object:?}/parameter/owner") - - if expr "${owner_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${owner_uid}" - else - printf '%s\n' "$(id -u -n "${owner_uid}")" - fi -fi +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 0d944482..d145c4c3 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*|11*) + 10*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http deleted file mode 100644 index 0508a465..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/http +++ /dev/null @@ -1,8 +0,0 @@ -frontend http - bind BIND@:80 - mode http - option httplog - default_backend http - -backend http - mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https deleted file mode 100644 index 73deac46..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/https +++ /dev/null @@ -1,10 +0,0 @@ -frontend https - bind BIND@:443 - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend https - -backend https - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps deleted file mode 100644 index b1ec3793..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/imaps +++ /dev/null @@ -1,12 +0,0 @@ -frontend imaps - bind BIND@:143 - bind BIND@:993 - - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend imaps - -backend imaps - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps deleted file mode 100644 index dce6ed4a..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/smtps +++ /dev/null @@ -1,12 +0,0 @@ -frontend smtps - bind BIND@:25 - bind BIND@:465 - - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend smtps - -backend smtps - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst deleted file mode 100644 index 6c131cbe..00000000 --- a/cdist/conf/type/__haproxy_dualstack/man.rst +++ /dev/null @@ -1,121 +0,0 @@ -cdist-type__haproxy_dualstack(7) -================================ - - -NAME ----- -cdist-type__haproxy_dualstack - Proxy services from a dual-stack server - - -DESCRIPTION ------------ -This (singleton) type installs and configures haproxy to act as a dual-stack -proxy for single-stack services. - -This can be useful to add IPv4 support to IPv6-only services while only using -one IPv4 for many such services. - -By default this type uses the plain TCP proxy mode, which means that there is no -need for TLS termination on this host when SNI is supported. -This also means that proxied services will not receive the client's IP address, -but will see the proxy's IP address instead (that of `$__target_host`). - -This can be solved by using the PROXY protocol, but do take into account that, -e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same -port, so you will need to use other ports for that. - -As a recommendation in this type: use TCP ports 8080 and 591 respectively to -serve HTTP and HTTPS using the PROXY protocol. - -See the EXAMPLES for more details. - - -OPTIONAL PARAMETERS -------------------- -v4proxy - Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. - In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is - the IP address actually providing the proxied services. - The full format of this argument is: - `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` - Where starting with `proxy:` determines that the PROXY protocol must be - used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT - override for the given PROTOCOL (see `--protocol`), if not present the - PROTOCOL's default port will be used. - - -v6proxy - Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. - In its simplest use, it must be a NAME with an `A` DNS entry, which is - the IP address actually providing the proxied services. - See `--v4proxy` for more options and details. - -protocol - Can be passed multiple times or as a space-separated list of protocols. - Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. - This defaults to: `http https imaps smtps`. - - -EXAMPLES --------- - -.. code-block:: sh - - # Proxy the IPv6-only services so IPv4-only clients can access them - # This uses HAProxy's TCP mode for http, https, imaps and smtps - __haproxy_dualstack \ - --v4proxy ipv6.chat \ - --v4proxy matrix.ungleich.ch - - # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them - # Note this means that the backend IPv6-only server will only see - # the IPv6 address of the haproxy host managed by cdist, which can be - # troublesome if this information is relevant for analytics/security/... - # See the PROXY example below - __haproxy_dualstack \ - --protocol http --protocol https \ - --v4proxy ipv6.chat \ - --v4proxy matrix.ungleich.ch - - # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling - # IPv4-only clients to access them while maintaining the client's IP address - __haproxy_dualstack \ - --protocol http --protocol https \ - --v4proxy proxy:ipv6.chat:http=8080:https=591 \ - --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 - # Note however that the PROXY protocol is not compatible with regular - # HTTP(S) protocols, so your nginx will have to listen on different ports - # with the PROXY settings. - # Note that you will need to restrict access to the 8080 port to prevent - # Client IP spoofing. - # This can be something like: - # server { - # # listen for regular HTTP connections - # listen [::]:80 default_server; - # listen 80 default_server; - # # listen for PROXY HTTP connections - # listen [::]:8080 proxy_protocol; - # # Accept the Client's IP from the PROXY protocol - # real_ip_header proxy_protocol; - # } - - -SEE ALSO --------- -- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ -- https://www.haproxy.com/blog/haproxy/proxy-protocol/ -- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ - - -AUTHORS -------- -ungleich -Evilham - - -COPYING -------- -Copyright \(C) 2021 ungleich glarus ag. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest deleted file mode 100644 index d110eea6..00000000 --- a/cdist/conf/type/__haproxy_dualstack/manifest +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/sh -eu - -__package haproxy -require="__package/haproxy" __start_on_boot haproxy - -tmpdir="$__object/files" -mkdir "$tmpdir" -configtmp="$__object/files/haproxy.cfg" - -os=$(cat "$__global/explorer/os") -case $os in - freebsd) - CONFIG_FILE="/usr/local/etc/haproxy.conf" - cat < "$configtmp" -global - maxconn 4000 - user nobody - group nogroup - daemon - -EOF - - ;; - *) - CONFIG_FILE="/etc/haproxy/haproxy.cfg" - cat < "$configtmp" -global - log [::1] local2 - chroot /var/lib/haproxy - pidfile /var/run/haproxy.pid - maxconn 4000 - user haproxy - group haproxy - daemon - - # turn on stats unix socket - stats socket /var/lib/haproxy/stats - -EOF - ;; -esac - -cat <> "$configtmp" -defaults - retries 3 - log global - timeout http-request 10s - timeout queue 1m - timeout connect 10s - timeout client 1m - timeout server 1m - timeout http-keep-alive 10s - timeout check 10s -EOF - -dig_cmd="$(command -v dig || true)" -get_ip() { - # Usage: get_ip (ipv4|ipv6) NAME - # uses "dig" if available, else fallback to "host" - case $1 in - ipv4) - if [ -n "${dig_cmd}" ]; then - ${dig_cmd} +short A "$2" - else - host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' - fi - ;; - ipv6) - if [ -n "${dig_cmd}" ]; then - ${dig_cmd} +short AAAA "$2" - else - host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' - fi - ;; - esac -} - -PROTOCOLS="$(cat "$__object/parameter/protocol")" - -for proxy in v4proxy v6proxy; do - param=$__object/parameter/$proxy - # no backend? skip generating code - if [ ! -f "$param" ]; then - continue - fi - - # turn backend name into bind parameter: v4backend -> ipv4@ - bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') - - case $bind in - ipv4) - backendproto=ipv6 - ;; - ipv6) - backendproto=ipv4 - ;; - esac - - for proto in ${PROTOCOLS}; do - # Add protocol "header" - printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" - - sed -e "s/BIND/$bind/" \ - -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ - -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ - "$__type/files/$proto" >> "$configtmp" - - while read -r hostdefinition; do - if echo "$hostdefinition" | grep -qE '^proxy:'; then - # Proxy protocol was requested - host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" - send_proxy=" send-proxy" - else - # Just use tcp proxy mode - host="$hostdefinition" - send_proxy="" - fi - if echo "$hostdefinition" | grep -qE ":${proto}="; then - # Use custom port definition if requested - port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" - else - # Else use the default - port="" - fi - servername=$host - - res=$(get_ip "$bind" "$servername") - - if [ -z "$res" ]; then - echo "$servername does not resolve - aborting config" >&2 - exit 1 - fi - - # Treat protocols without TLS+SNI specially - if [ "$proto" = http ]; then - echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" - else - echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" - fi - - # Create the "server" itself. - # Note that port and send_proxy will be empty unless - # they were requested by the type user - echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" - - done < "$param" - done -done - -# Create config file -require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 - -require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ - --pattern "^__file${CONFIG_FILE}" \ - --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol deleted file mode 100644 index dc8bb7bf..00000000 --- a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol +++ /dev/null @@ -1 +0,0 @@ -http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple deleted file mode 100644 index 8c482bd4..00000000 --- a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple +++ /dev/null @@ -1,3 +0,0 @@ -protocol -v4proxy -v6proxy diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path new file mode 100755 index 00000000..3c6076df --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path @@ -0,0 +1,3 @@ +#!/bin/sh -e + +command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data deleted file mode 100755 index ff62e742..00000000 --- a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-data +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/sh -e -certbot_path="$(command -v certbot 2>/dev/null || true)" -# Defaults -certificate_exists="no" -certificate_is_test="no" - -if [ -n "${certbot_path}" ]; then - # Find python executable that has access to certbot's module - python_path=$(sed -n '1s/^#! *//p' "${certbot_path}") - - # Use a lock for cdist due to certbot not exiting with failure - # or having any flags for concurrent use. - _certbot() { - ${python_path} - 2>/dev/null <> /dev/stderr - exit 1 - ;; - esac - - hook_contents_tail="$(cat < "${existing_domains}" - certificate_is_test="$(_explorer_var certificate_is_test)" + existing_domains="${__object}/explorer/certificate-domains" + certificate_is_test=$(cat "${__object}/explorer/certificate-is-test") sort -uo "${requested_domains}" "${requested_domains}" sort -uo "${existing_domains}" "${existing_domains}" diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst index 43be8424..85eb88ea 100644 --- a/cdist/conf/type/__letsencrypt_cert/man.rst +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -1,33 +1,16 @@ cdist-type__letsencrypt_cert(7) =============================== - NAME ---- cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt - DESCRIPTION ----------- Automatically obtain a Let's Encrypt SSL certificate using Certbot. -This type attempts to setup automatic renewals always. In many Linux -distributions, that is the case out of the box, see: -https://certbot.eff.org/docs/using.html#automated-renewals - -For Alpine Linux and Arch Linux, we setup a system-wide cronjob that -attempts to renew certificates daily. - -If you are using FreeBSD, we configure periodic(8) as recommended by -the port mantainer, so there will be a weekly attempt at renewal. - -If your OS is not mentioned here or on Certbot's docs as having -support for automated renewals, please make sure you check your OS -and possibly patch this type so the system-wide cronjob is installed. - - REQUIRED PARAMETERS ------------------- @@ -38,7 +21,6 @@ object id admin-email Where to send Let's Encrypt emails like "certificate needs renewal". - OPTIONAL PARAMETERS ------------------- @@ -54,68 +36,25 @@ webroot The path to your webroot, as set up in your webserver config. If this parameter is not present, Certbot will be run in standalone mode. - OPTIONAL MULTIPLE PARAMETERS ---------------------------- +renew-hook + Renew hook command directly passed to Certbot in cron job. + domain Domains to be included in the certificate. When specified then object id is not used as a domain. -deploy-hook - Command to be executed only when the certificate associated with this - ``$__object_id`` is issued or renewed. - You can specify it multiple times, but any failure will prevent further - commands from being executed. - - For this command, the - shell variable ``$RENEWED_LINEAGE`` will point to the - config live subdirectory (for example, - ``/etc/letsencrypt/live/${__object_id}``) containing the - new certificates and keys; the shell variable - ``$RENEWED_DOMAINS`` will contain a space-delimited list - of renewed certificate domains (for example, - ``example.com www.example.com``) - -pre-hook - Command to be run in a shell before obtaining any - certificates. - You can specify it multiple times, but any failure will prevent further - commands from being executed. - - Note these run regardless of which certificate is attempted, you may want to - manage these system-wide hooks with ``__file`` in - ``/etc/letsencrypt/renewal-hooks/pre/``. - - Intended primarily for renewal, where it - can be used to temporarily shut down a webserver that - might conflict with the standalone plugin. This will - only be called if a certificate is actually to be - obtained/renewed. - -post-hook - Command to be run in a shell after attempting to - obtain/renew certificates. - You can specify it multiple times, but any failure will prevent further - commands from being executed. - - Note these run regardless of which certificate was attempted, you may want to - manage these system-wide hooks with ``__file`` in - ``/etc/letsencrypt/renewal-hooks/post/``. - - Can be used to deploy - renewed certificates, or to restart any servers that - were stopped by --pre-hook. This is only run if an - attempt was made to obtain/renew a certificate. - - BOOLEAN PARAMETERS ------------------ +automatic-renewal + Install a cron job, which attempts to renew certificates daily. + staging Obtain a test certificate from a staging server. - MESSAGES -------- @@ -128,7 +67,6 @@ create remove Certificate was removed. - EXAMPLES -------- @@ -137,7 +75,8 @@ EXAMPLES # use object id as domain __letsencrypt_cert example.com \ --admin-email root@example.com \ - --deploy-hook "service nginx reload" \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ --webroot /data/letsencrypt/root .. code-block:: sh @@ -146,10 +85,11 @@ EXAMPLES # and example.com needs to be included again with domain parameter __letsencrypt_cert example.com \ --admin-email root@example.com \ + --automatic-renewal \ --domain example.com \ --domain foo.example.com \ --domain bar.example.com \ - --deploy-hook "service nginx reload" \ + --renew-hook "service nginx reload" \ --webroot /data/letsencrypt/root AUTHORS @@ -159,13 +99,11 @@ AUTHORS | Kamila Součková | Darko Poljak | Ľubomír Kučera -| Evilham - COPYING ------- -Copyright \(C) 2017-2021 Nico Schottelius, Kamila Součková, Darko Poljak and +Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and Ľubomír Kučera. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest old mode 100644 new mode 100755 index 638a99e0..b4464366 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -1,20 +1,18 @@ #!/bin/sh -certbot_fullpath="$(grep "^certbot_path:" "${__object:?}/explorer/certificate-data" | cut -d ':' -f 2-)" -state=$(cat "${__object}/parameter/state") -os="$(cat "${__global:?}/explorer/os")" +certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" if [ -z "${certbot_fullpath}" ]; then + os="$(cat "${__global:?}/explorer/os")" os_version="$(cat "${__global}/explorer/os_version")" - # Use this, very common value, as a default. It is OS-dependent - certbot_fullpath="/usr/bin/certbot" + case "$os" in - archlinux) - __package certbot - ;; - alpine) - __package certbot - ;; + archlinux) + __package certbot + ;; + alpine) + __package certbot + ;; debian) case "$os_version" in 8*) @@ -41,7 +39,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*|11*) + 10*) __package_apt certbot ;; @@ -50,7 +48,9 @@ if [ -z "${certbot_fullpath}" ]; then exit 1 ;; esac - ;; + + certbot_fullpath=/usr/bin/certbot + ;; devuan) case "$os_version" in jessie) @@ -83,14 +83,17 @@ if [ -z "${certbot_fullpath}" ]; then exit 1 ;; esac + + certbot_fullpath=/usr/bin/certbot ;; freebsd) - __package py37-certbot - certbot_fullpath="/usr/local/bin/certbot" + __package py27-certbot + + certbot_fullpath=/usr/local/bin/certbot ;; ubuntu) - __package certbot - ;; + __package certbot + ;; *) echo "Unsupported os: $os" >&2 exit 1 @@ -98,61 +101,18 @@ if [ -z "${certbot_fullpath}" ]; then esac fi -# Other OS-dependent values that we want to set every time -LE_DIR="/etc/letsencrypt" -certbot_cronjob_state="absent" -case "$os" in - archlinux|alpine) - certbot_cronjob_state="present" - ;; - freebsd) - LE_DIR="/usr/local/etc/letsencrypt" - # FreeBSD uses periodic(8) instead of crontabs for this - __line "periodic.conf_weekly_certbot" \ - --file "/etc/periodic.conf" \ - --regex "^(#[[:space:]]*)?weekly_certbot_enable=.*" \ - --state "replace" \ - --line 'weekly_certbot_enable="YES"' - ;; - *) - ;; -esac +if [ -f "${__object}/parameter/automatic-renewal" ]; then + renew_hook_param="${__object}/parameter/renew-hook" + renew_hook="" + if [ -f "${renew_hook_param}" ]; then + while read -r hook; do + renew_hook="${renew_hook} --renew-hook \"${hook}\"" + done < "${renew_hook_param}" + fi -# This is only necessary in certain OS -__cron letsencrypt-certbot \ - --user root \ - --command "${certbot_fullpath} renew -q" \ - --hour 0 \ - --minute 47 \ - --state "${certbot_cronjob_state}" - -# Ensure hook directories -HOOKS_DIR="${LE_DIR}/renewal-hooks" -__directory "${LE_DIR}" --mode 0755 -require="__directory/${LE_DIR}" __directory "${HOOKS_DIR}" --mode 0755 - -if [ -f "${__object}/parameter/domain" ]; then - domains="$(sort "${__object}/parameter/domain")" -else - domains="${__object_id}" + __cron letsencrypt-certbot \ + --user root \ + --command "${certbot_fullpath} renew -q ${renew_hook}" \ + --hour 0 \ + --minute 47 fi - -# Install hooks as needed -for hook in deploy pre post; do - # Using something unique and specific to this object - hook_file="${HOOKS_DIR}/${hook}/${__object_id}.cdist.sh" - - # This defines hook_contents - # shellcheck source=cdist/conf/type/__letsencrypt_cert/files/gen_hook.sh - . "${__type}/files/gen_hook.sh" - - # Ensure hook directory exists - require="__directory/${HOOKS_DIR}" __directory "${HOOKS_DIR}/${hook}" \ - --mode 0755 - require="__directory/${HOOKS_DIR}/${hook}" __file "${hook_file}" \ - --mode 0555 \ - --source '-' \ - --state "${hook_state}" < /dev/null 2>&1 || true +then echo apt-get update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then diff --git a/cdist/conf/type/__package_pip/explorer/distinfo-dir b/cdist/conf/type/__package_pip/explorer/distinfo-dir deleted file mode 100755 index 18e169ae..00000000 --- a/cdist/conf/type/__package_pip/explorer/distinfo-dir +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh -# -# 2021 Matthias Stecher (matthiasstecher at gmx.de) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - - -nameparam="$__object/parameter/name" -if [ -f "$nameparam" ]; then - name=$(cat "$nameparam") -else - name="$__object_id" -fi - -pipparam="$__object/parameter/pip" -if [ -f "$pipparam" ]; then - pip=$(cat "$pipparam") -else - pip="$( "$__type_explorer/pip" )" -fi - - -if command -v "$pip" >/dev/null 2>&1; then - # assemble the path where pip stores all pip package info - "$pip" show "$name" \ - | awk -F': ' ' - $1 == "Name" {name=$2; gsub(/-/,"_",name); next} - $1 == "Version" {version=$2; next} - $1 == "Location" {location=$2; next} - END {if (version != "") printf "%s/%s-%s.dist-info", location, name, version}' -fi diff --git a/cdist/conf/type/__package_pip/explorer/extras b/cdist/conf/type/__package_pip/explorer/extras deleted file mode 100755 index bbdc17ab..00000000 --- a/cdist/conf/type/__package_pip/explorer/extras +++ /dev/null @@ -1,66 +0,0 @@ -#!/bin/sh -# -# 2021 Matthias Stecher (matthiasstecher at gmx.de) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Checks if the given extras are really installed or not. It will be -# done by querring all dependencies for that extra and return it as -# "to be installed" if no dependency was found. -# - - -distinfo_dir="$("$__type_explorer/distinfo-dir")" - -# check if we have something to check -if [ "$distinfo_dir" ] && [ -s "$__object/parameter/extra" ] -then - # save cause freezing is slow - mkdir "$__object/files" - pip_freeze="$__object/files/pip-freeze.tmp" - pip3 freeze > "$pip_freeze" - - # If all is set, it searches all available extras to separatly check them. - # It would work with just 'all' (cause dependencies are specified for - # 'all'), but will not update if one extra is already present. Side effect - # is that it will not use [all] but instead name all extras seperatly. - for extra in $(if grep -qFx all "$__object/parameter/extra"; - then awk -F': ' '$1 == "Provides-Extra" && $2 != "all"{print $2}' "$distinfo_dir/METADATA"; - else tr ',' '\n' < "$__object/parameter/extra"; - fi) - do - # create a grep BRE pattern to search all packages - # maybe a file full of patterns for -F could be written - grep_pattern="$( - awk -F'(: | ; )' -v check="$extra" ' - $1 == "Requires-Dist" { - split($2, r, " "); - sub("extra == ", "", $3); gsub("'"'"'", "", $3); - if($3 == check) print r[1] - }' "$distinfo_dir/METADATA" \ - | sed ':a; $!N; s/\n/\\|/; ta' - )" - - # echo the extra if no packages where found for it - # if there is no pattern, we don't need to search ;-) - # pip matches packages case-insensetive, we need to do that, too - if [ "$grep_pattern" ] && ! grep -qi "$grep_pattern" "$pip_freeze" - then - echo "$extra" - fi - done -fi diff --git a/cdist/conf/type/__package_pip/explorer/state b/cdist/conf/type/__package_pip/explorer/state old mode 100755 new mode 100644 diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote index 9abe28bf..a1375c2d 100755 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -2,7 +2,6 @@ # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) -# 2021 Matthias Stecher (matthiasstecher at gmx.de) # # This file is part of cdist. # @@ -26,10 +25,7 @@ state_is=$(cat "$__object/explorer/state") state_should="$(cat "$__object/parameter/state")" -# short circuit if state is the same and no extras to install -[ "$state_is" = "$state_should" ] && ! [ -s "$__object/explorer/extras" ] \ - && exit 0 - +[ "$state_is" = "$state_should" ] && exit 0 nameparam="$__object/parameter/name" if [ -f "$nameparam" ]; then @@ -60,14 +56,6 @@ fi case "$state_should" in present) - if [ -s "$__object/explorer/extras" ] - then - # all extras are passed to pip in a comma-separated list in the name - # sed loops through all input lines and add commas between them - extras="$(sed ':a; $!N; s/\n/,/; ta' "$__object/explorer/extras")" - name="${name}[${extras}]" - fi - if [ "$runas" ] then echo "su -c '$pip install -q $name' $runas" diff --git a/cdist/conf/type/__package_pip/man.rst b/cdist/conf/type/__package_pip/man.rst index 5a2bc673..234ceee2 100644 --- a/cdist/conf/type/__package_pip/man.rst +++ b/cdist/conf/type/__package_pip/man.rst @@ -22,16 +22,6 @@ OPTIONAL PARAMETERS name If supplied, use the name and not the object id as the package name. -extra - Extra optional dependencies which should be installed along the selected - package. Can be specified multiple times. Multiple extras can be passed - in one `--extra` as a comma-separated list. - - Extra optional dependencies will be installed even when the base package - is already installed. Notice that the type will not remove installed extras - that are not explicitly named for the type because pip does not offer a - management for orphaned packages and they may be used by other packages. - pip Instead of using pip from PATH, use the specific pip path. @@ -56,14 +46,6 @@ EXAMPLES # Use pip in a virtualenv located at /foo/shinken_virtualenv as user foo __package_pip pyro --state present --pip /foo/shinken_virtualenv/bin/pip --runas foo - # Install package with optional dependencies - __package_pip mautrix-telegram --extra speedups --extra webp_convert --extra hq_thumbnails - # the extras can also be specified comma-separated - __package_pip mautrix-telegram --extra speedups,webp_convert,hq_thumbnails --extra postgres - - # or take all extras - __package_pip mautrix-telegram --extra all - SEE ALSO -------- @@ -72,13 +54,12 @@ SEE ALSO AUTHORS ------- -| Nico Schottelius -| Matthias Stecher +Nico Schottelius COPYING ------- -Copyright \(C) 2012 Nico Schottelius, 2021 Matthias Stecher. You can -redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either -version 3 of the License, or (at your option) any later version. +Copyright \(C) 2012 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__package_pip/parameter/optional_multiple b/cdist/conf/type/__package_pip/parameter/optional_multiple deleted file mode 100644 index 0f228715..00000000 --- a/cdist/conf/type/__package_pip/parameter/optional_multiple +++ /dev/null @@ -1 +0,0 @@ -extra diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index ca9aa45a..3f88f6bc 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -37,7 +37,6 @@ assert () # If condition false, then echo "Assertion failed: \"$1\"" # shellcheck disable=SC2039 - # shellcheck disable=SC3044 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED fi diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index a10c16d3..803468b5 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,19 +41,7 @@ fi case "$type" in yum) ;; apt) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - - echo "apt-get --quiet $apt_opts update" + echo "apt-get --quiet update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index d332e851..38aa001e 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,10 +28,6 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" -if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then - apt_with_new_pkgs="--with-new-pkgs" -fi - if [ -f "$type" ]; then type="$(cat "$type")" else @@ -58,7 +54,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget $apt_with_new_pkgs upgrade" + else echo "$aptget upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index 0c116bac..e9e2b8ce 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,14 +33,6 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. -apt-with-new-pkg - Allow installing new packages when used in conjunction with - upgrade. This is useful if the update of an installed package - requires new dependencies to be installed. Instead of holding the - package back upgrade will upgrade the package and install the new - dependencies. Note that upgrade with this option will never remove - packages, only allow adding new ones. - apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index cd22eb90..7a56a34b 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,3 +1,2 @@ apt-clean apt-dist-upgrade -apt-with-new-pkgs diff --git a/cdist/conf/type/__postgres_conf/explorer/postgres_user b/cdist/conf/type/__postgres_conf/explorer/postgres_user deleted file mode 100644 index c6582dc4..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/postgres_user +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -os=$("${__explorer:?}/os") - -case ${os} -in - (alpine) - echo 'postgres' - ;; - (centos|rhel|scientific) - echo 'postgres' - ;; - (debian|devuan|ubuntu) - echo 'postgres' - ;; - (freebsd) - test -x /usr/local/etc/rc.d/postgresql || { - printf 'could not find postgresql rc script./n' >&2 - exit 1 - } - pg_status=$(/usr/local/etc/rc.d/postgresql onestatus) || { - printf 'postgresql daemon is not running.\n' >&2 - exit 1 - } - pg_pid=$(printf '%s\n' "${pg_status}" \ - | sed -n 's/^pg_ctl:.*(PID: *\([0-9]*\))$/\1/p') - - # PostgreSQL < 9.6: pgsql - # PostgreSQL >= 9.6: postgres - ps -o user -p "${pg_pid}" | sed -n '2p' - ;; - (netbsd) - echo 'pgsql' - ;; - (openbsd) - echo '_postgresql' - ;; - (suse) - echo 'postgres' - ;; - (*) - echo "Unsupported OS: ${os}" >&2 - exit 1 - ;; -esac diff --git a/cdist/conf/type/__postgres_conf/explorer/state b/cdist/conf/type/__postgres_conf/explorer/state deleted file mode 100644 index 4b7b0a43..00000000 --- a/cdist/conf/type/__postgres_conf/explorer/state +++ /dev/null @@ -1,223 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -postgres_user=$("${__type_explorer:?}/postgres_user") -conf_name=${__object_id:?} - -tolower() { printf '%s' "$*" | tr '[:upper:]' '[:lower:]'; } - -tobytes() { - # NOTE: This function treats everything as base 2. - # It is not compatible with SI units. - awk 'BEGIN { FS = "\n" } - /TB$/ { $0 = ($0 * 1024) "GB" } - /GB$/ { $0 = ($0 * 1024) "MB" } - /MB$/ { $0 = ($0 * 1024) "kB" } - /kB$/ { $0 = ($0 * 1024) "B" } - /B?$/ { sub(/ *B?$/, "") } - ($0*1) == $0 # is number - ' <<-EOF - $1 - EOF -} - -tomillisecs() { - awk 'BEGIN { FS = "\n" } - /d$/ { $0 = ($0 * 24) "h" } - /h$/ { $0 = ($0 * 60) "min" } - /min$/ { $0 = ($0 * 60) "s" } - /[^m]s$/ { $0 = ($0 * 1000) "ms" } - /ms$/ { $0 *= 1 } - ($0*1) == $0 # is number - ' <<-EOF - $1 - EOF -} - -tobool() { - # prints either 'on' or 'off' - case $(tolower "$1") - in - (t|true|y|yes|on|1) - echo 'on' ;; - (f|false|n|no|off|0) - echo 'off' ;; - (*) - printf 'Inavlid bool value: %s\n' "$2" >&2 - return 1 - ;; - esac - return 0 -} - -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } -psql_exec() { - su - "${postgres_user}" -c "psql postgres -twAc $(quote "$*")" -} - -psql_conf_source() { - # NOTE: SHOW/SET are case-insentitive, so this command should also be. - psql_exec "SELECT CASE WHEN source = 'default' OR setting = boot_val THEN 'default' ELSE source END FROM pg_settings WHERE lower(name) = lower('$1')" -} -psql_conf_cmp() ( - IFS='|' read -r lower_name vartype setting unit <<-EOF - $(psql_exec "SELECT lower(name), vartype, setting, unit FROM pg_settings WHERE lower(name) = lower('$1')") - EOF - - should_value=$2 - is_value=${setting} - - # The following case contains special cases for special settings. - case ${lower_name} - in - (archive_command) - if test "${setting}" = '(disabled)' - then - # DAFUQ PostgreSQL?! - # PostgreSQL returns (disabled) if the feature is inactive. - # We cannot compare the values unless it is enabled, first. - return 0 - fi - ;; - (archive_mode|backslash_quote|constraint_exclusion|force_parallel_mode|huge_pages|synchronous_commit) - # Although only 'on', 'off' are documented, PostgreSQL accepts all - # the "likely" variants of "on" and "off". - case $(tolower "${should_value}") - in - (on|off|true|false|yes|no|1|0) - should_value=$(tobool "${should_value}") - ;; - esac - ;; - esac - - case ${vartype} - in - (bool) - test -z "${unit}" || { - # please fix the explorer if this error occurs. - printf 'units are not supported for vartype: %s\n' "${vartype}" >&2 - exit 1 - } - - should_value=$(tobool "${should_value}") - - test "${is_value}" = "${should_value}" - ;; - (enum) - test -z "${unit}" || { - # please fix the explorer if this error occurs. - printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 - exit 1 - } - - # NOTE: All enums that are currently defined are lower case, but - # PostgreSQL also accepts upper case spelling. - should_value=$(tolower "$2") - - test "${is_value}" = "${should_value}" - ;; - (integer) - # split multiples from unit, first (e.g. 8kB -> 8, kB) - case ${unit} - in - ([0-9]*) - multiple=${unit%%[!0-9]*} - unit=${unit##*[0-9 ]} - ;; - (*) multiple=1 ;; - esac - - is_value=$((setting * multiple))${unit} - - if expr "${should_value}" : '-\{0,1\}[0-9]*$' >/dev/null - then - # default unit - should_value=$((should_value * multiple))${unit} - fi - - # then, do conversion - # NOTE: these conversions work for integers only! - case ${unit} - in - (B|[kMGT]B) - # bytes - is_bytes=$(tobytes "${is_value}") - should_bytes=$(tobytes "${should_value}") - - test $((is_bytes)) -eq $((should_bytes)) - ;; - (ms|s|min|h|d) - # seconds - is_ms=$(tomillisecs "${is_value}") - should_ms=$(tomillisecs "${should_value}") - - test $((is_ms)) -eq $((should_ms)) - ;; - ('') - # no unit - is_int=${is_value} - should_int=${should_value} - - test $((is_int)) -eq $((should_int)) - ;; - esac - ;; - (real|string) - # NOTE: reals could possibly have units, but currently there none. - - test -z "${unit}" || { - # please fix the explorer if this error occurs. - printf 'units are not supported with vartype: %s\n' "${vartype}" >&2 - exit 1 - } - - test "${is_value}" = "${should_value}" - ;; - esac -) - -psql_exec 'SELECT 1' >/dev/null || { - echo 'Connection to PostgreSQL server failed' >&2 - exit 1 -} - -case $(psql_conf_source "${conf_name}") -in - ('') - printf 'Invalid configuration parameter: %s\n' "${conf_name}" >&2 - exit 1 - ;; - (default) - echo absent - ;; - (*) - if ! test -f "${__object:?}/parameter/value" - then - echo present - elif psql_conf_cmp "${conf_name}" "$(cat "${__object:?}/parameter/value")" - then - echo present - else - echo different - fi - ;; -esac diff --git a/cdist/conf/type/__postgres_conf/gencode-remote b/cdist/conf/type/__postgres_conf/gencode-remote deleted file mode 100755 index 27651600..00000000 --- a/cdist/conf/type/__postgres_conf/gencode-remote +++ /dev/null @@ -1,123 +0,0 @@ -#!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- -# -# 2019-2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) -# 2020 Beni Ruef (bernhard.ruef at ssrq-sds-fds.ch) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -state_is=$(cat "${__object:?}/explorer/state") -state_should=$(cat "${__object:?}/parameter/state") -postgres_user=$(cat "${__object:?}/explorer/postgres_user") - -conf_name=${__object_id:?} - -if test "${state_is}" = "${state_should}" -then - exit 0 -fi - -quote() { - for _arg - do - shift - if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" - then - # needs quoting - set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" - else - set -- "$@" "${_arg}" - fi - done - unset _arg - - # NOTE: Use printf because POSIX echo interprets escape sequences - printf '%s' "$*" -} - - -psql_cmd() { - printf 'su - %s -c %s\n' "$(quote "${postgres_user}")" "$(quote "$(quote psql "$@")")" -} - -case ${state_should} -in - (present) - test -n "${__object:?}/parameter/value" || { - echo 'Missing required parameter --value' >&2 - exit 1 - } - - cat <<-EOF - exec 3< "\${__object:?}/parameter/value" - $(psql_cmd postgres -tAwq -o /dev/null -v ON_ERROR_STOP=on) <<'SQL' - \\set conf_value \`cat <&3\` - ALTER SYSTEM SET ${conf_name} = :'conf_value'; - SELECT pg_reload_conf(); - SQL - exec 3<&- - EOF - ;; - (absent) - psql_cmd postgres -qwc "ALTER SYSTEM SET ${conf_name} TO DEFAULT" - ;; - (*) - printf 'Invalid --state: %s\n' "${state_should}" >&2 - printf 'Only "present" and "absent" are acceptable.\n' >&2 - exit 1 - ;; -esac - -# Restart PostgreSQL server if required to apply new configuration value -cat <&2 - exit 1 - esac - ;; - (*) - printf "Don't know how to restart services with your init (%s)\n" "${init}" >&2 - exit 1 - esac - ) -fi -EOF diff --git a/cdist/conf/type/__postgres_conf/man.rst b/cdist/conf/type/__postgres_conf/man.rst deleted file mode 100644 index e035f080..00000000 --- a/cdist/conf/type/__postgres_conf/man.rst +++ /dev/null @@ -1,60 +0,0 @@ -cdist-type__postgres_conf(7) -============================ - -NAME ----- -cdist-type__postgres_conf - Alter PostgreSQL configuration - - -DESCRIPTION ------------ -Configure a running PostgreSQL server using ``ALTER SYSTEM``. - - -REQUIRED PARAMETERS -------------------- -value - The value to set (can be omitted if ``--state`` is set to ``absent``). - - -OPTIONAL PARAMETERS -------------------- -state - ``present`` or ``absent``. - Defaults to ``present``. - - -BOOLEAN PARAMETERS ------------------- -None. - - -EXAMPLES --------- - -.. code-block:: sh - - # set timezone - __postgres_conf timezone --value Europe/Zurich - - # reset maximum number of concurrent connections to default (normally 100) - __postgres_conf max_connections --state absent - - -SEE ALSO --------- -None. - - -AUTHORS -------- -Beni Ruef (bernhard.ruef--@--ssrq-sds-fds.ch) -Dennis Camera (dennis.camera--@--ssrq-sds-fds.ch) - - -COPYING -------- -Copyright \(C) 2019-2021 SSRQ (www.ssrq-sds-fds.ch). -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. diff --git a/cdist/conf/type/__postgres_conf/parameter/default/state b/cdist/conf/type/__postgres_conf/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__postgres_conf/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__postgres_conf/parameter/optional b/cdist/conf/type/__postgres_conf/parameter/optional deleted file mode 100644 index d0460d86..00000000 --- a/cdist/conf/type/__postgres_conf/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -state -value diff --git a/cdist/conf/type/__postgres_database/explorer/postgres_user b/cdist/conf/type/__postgres_database/explorer/postgres_user deleted file mode 120000 index 714e7237..00000000 --- a/cdist/conf/type/__postgres_database/explorer/postgres_user +++ /dev/null @@ -1 +0,0 @@ -../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index 6a25df86..d68d4120 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -1,7 +1,6 @@ #!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,18 +18,25 @@ # along with cdist. If not, see . # -postgres_user=$("${__type_explorer:?}/postgres_user") +case "$("${__explorer}/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac -dbname=${__object_id:?} -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } -psql_exec() { - su - "${postgres_user}" -c "psql $(quote "$1") -twAc $(quote "$2")" -} +name="$__object_id" -if psql_exec postgres "SELECT datname FROM pg_database" | grep -qFx "${dbname}" +if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" then - echo 'present' + echo 'present' else - echo 'absent' + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index 7d7d6fa2..0f11cff4 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -19,63 +18,60 @@ # along with cdist. If not, see . # -quote() { - for _arg - do - shift - if test -n "$(printf '%s' "${_arg}" | tr -d -c '\t\n \042-\047\050-\052\073-\077\133\\`|~' | tr -c '' '.')" - then - # needs quoting - set -- "$@" "'$(printf '%s' "${_arg}" | sed -e "s/'/'\\\\''/g")'" - else - set -- "$@" "${_arg}" - fi - done - unset _arg - - # NOTE: Use printf because POSIX echo interprets escape sequences - printf '%s' "$*" -} - -postgres_user=$(cat "${__object:?}/explorer/postgres_user") - -dbname=${__object_id:?} -state_should=$(cat "${__object:?}/parameter/state") -state_is=$(cat "${__object:?}/explorer/state") - -if test "${state_should}" = "$state_is" -then - exit 0 -fi - -case ${state_should} +case "$(cat "${__global}/explorer/os")" in - (present) - set -- - - while read -r param_name opt - do - if test -f "${__object:?}/parameter/${param_name}" - then - set -- "$@" "${opt}" "$(cat "${__object:?}/parameter/${param_name}")" - fi - done <<-'EOF' - owner -O - template --template - encoding --encoding - lc_collate --lc-collate - lc_ctype --lc-ctype - EOF - - set -- "$@" "${dbname}" - - cat <<-EOF - su - $(quote "${postgres_user}") -c $(quote "$(quote createdb "$@")") - EOF - ;; - (absent) - cat <<-EOF - su - $(quote "${postgres_user}") -c $(quote "$(quote dropdb "${dbname}")") - EOF - ;; + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; esac + + +name="$__object_id" +state_should="$(cat "$__object/parameter/state")" +state_is="$(cat "$__object/explorer/state")" + +if [ "$state_should" != "$state_is" ]; then + case "$state_should" in + present) + owner="" + if [ -f "$__object/parameter/owner" ]; then + owner="-O \"$(cat "$__object/parameter/owner")\"" + fi + + template="" + if [ -f "$__object/parameter/template" ]; then + template="--template \"$(cat "$__object/parameter/template")\"" + fi + + encoding="" + if [ -f "$__object/parameter/encoding" ]; then + encoding="--encoding \"$(cat "$__object/parameter/encoding")\"" + fi + + lc_collate="" + if [ -f "$__object/parameter/lc-collate" ]; then + lc_collate="--lc-collate \"$(cat "$__object/parameter/lc-collate")\"" + fi + + lc_ctype="" + if [ -f "$__object/parameter/lc-ctype" ]; then + lc_ctype="--lc-ctype \"$(cat "$__object/parameter/lc-ctype")\"" + fi + + cat << EOF +su - '$postgres_user' -c "createdb $owner \"$name\" $template $encoding $lc_collate $lc_ctype" +EOF + ;; + absent) + cat << EOF +su - '$postgres_user' -c "dropdb \"$name\"" +EOF + ;; + esac +fi diff --git a/cdist/conf/type/__postgres_extension/explorer/postgres_user b/cdist/conf/type/__postgres_extension/explorer/postgres_user deleted file mode 120000 index 714e7237..00000000 --- a/cdist/conf/type/__postgres_extension/explorer/postgres_user +++ /dev/null @@ -1 +0,0 @@ -../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote index 4ce72622..af9c97f1 100755 --- a/cdist/conf/type/__postgres_extension/gencode-remote +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -2,10 +2,9 @@ # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch) -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This type was created by Tomas Pospisek based on the -# __postgres_role type by Steven Armstrong. +#__postgres_role type by Steven Armstrong # # This file is part of cdist. # @@ -23,38 +22,32 @@ # along with cdist. If not, see . # -postgres_user=$(cat "${__object:?}/explorer/postgres_user") - -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } -psql_cmd() { - printf 'su - %s -c %s\n' \ - "$(quote "${postgres_user}")" \ - "$(quote psql "$(quote "$1")" -c "$(quote "$2")")" -} - - -IFS=: read -r dbname extname <&2 - exit 1 - ;; + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + +dbname=$( echo "$__object_id" | cut -d":" -f1 ) +extension=$( echo "$__object_id" | cut -d":" -f2 ) + +state_should=$( cat "$__object/parameter/state" ) + +case "$state_should" in + present) + cmd="CREATE EXTENSION IF NOT EXISTS $extension" + echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'" + ;; + absent) + cmd="DROP EXTENSION IF EXISTS $extension" + echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'" + ;; esac diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 442239f6..79645b2b 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -3,36 +3,32 @@ cdist-type__postgres_extension(7) NAME ---- -cdist-type__postgres_extension - Manage PostgreSQL extensions +cdist-type__postgres_extension - manage postgres extensions DESCRIPTION ----------- -This cdist type allows you to manage PostgreSQL extensions. +This cdist type allows you to create or drop postgres extensions. -The ``__object_id`` to pass to ``__postgres_extension`` is of the form -``dbname:extension``, e.g.: +The object you need to pass to __postgres_extension consists of +the database name and the extension name joined by a colon in the +following form: + +.. code-block:: sh + + dbname:extension + +f.ex. .. code-block:: sh rails_test:unaccent -**CAUTION!** Be careful when installing extensions from (untrusted) third-party -sources: - - | Installing an extension as superuser requires trusting that the extension's - author wrote the extension installation script in a secure fashion. It is - not terribly difficult for a malicious user to create trojan-horse objects - that will compromise later execution of a carelessly-written extension - script, allowing that user to acquire superuser privileges. - | – ``_ - - OPTIONAL PARAMETERS ------------------- state - either ``present`` or ``absent``, defaults to ``present``. + either "present" or "absent", defaults to "present" EXAMPLES @@ -40,29 +36,24 @@ EXAMPLES .. code-block:: sh - # Install extension unaccent into database rails_test - __postgres_extension rails_test:unaccent - - # Drop extension unaccent from database fails_test - __postgres_extension rails_test:unaccent --state absent + __postgres_extension rails_test:unaccent + __postgres_extension --present rails_test:unaccent + __postgres_extension --absent rails_test:unaccent SEE ALSO -------- -- :strong:`cdist-type__postgres_database`\ (7) -- PostgreSQL "CREATE EXTENSION" documentation at: - ``_. +:strong:`cdist-type__postgre_database`\ (7) +Postgres "Create Extension" documentation at: . -AUTHORS +AUTHOR ------- -| Tomas Pospisek -| Dennis Camera - +Tomas Pospisek COPYING ------- -Copyright \(C) 2014 Tomas Pospisek, 2021 Dennis Camera. -You can redistribute it and/or modify it under the terms of the GNU General -Public License as published by the Free Software Foundation, either version 3 of -the License, or (at your option) any later version. +Copyright \(C) 2014 Tomas Pospisek. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__postgres_role/explorer/postgres_user b/cdist/conf/type/__postgres_role/explorer/postgres_user deleted file mode 120000 index 714e7237..00000000 --- a/cdist/conf/type/__postgres_role/explorer/postgres_user +++ /dev/null @@ -1 +0,0 @@ -../../__postgres_conf/explorer/postgres_user \ No newline at end of file diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 822816c1..c8e1fa9d 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -1,7 +1,6 @@ -#!/bin/sh -e +#!/bin/sh # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -12,125 +11,32 @@ # # cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -postgres_user=$("${__type_explorer:?}/postgres_user") -rolename=${__object_id:?} +case "$("${__explorer}/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac -psql_query() { - su -l "${postgres_user}" -c "$( - printf "psql -q -F '\034' -R '\036' -wAc '%s'" \ - "$(printf %s "$*" | sed "s/'/'\\\\''/g")" - )" -} +name="$__object_id" -password_check_login() ( - PGPASSWORD=$(cat "${__object:?}/parameter/password"; printf .) - PGPASSWORD=${PGPASSWORD%?.} - export PGPASSWORD - psql -q -w -h localhost -U "${rolename}" template1 -c '\q' >/dev/null 2>&1 -) - -role_properties=$( - psql_query "SELECT * FROM pg_roles WHERE rolname = '${rolename}'" \ - | awk ' - BEGIN { RS = "\036"; FS = "\034" } - /^\([0-9]+ rows?\)/ { exit } - NR == 1 { for (i = 1; i <= NF; i++) cols[i] = $i; next } - NR == 2 { for (i = 1; i <= NF; i++) printf "%s=%s\n", cols[i], $i }' -) - -if test -n "${role_properties}" +if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")" then - # Check if the user's properties match the parameters - for prop in login createdb createrole superuser - do - bool_should=$(test -f "${__object:?}/parameter/${prop}" && echo 't' || echo 'f') - bool_is=$( - printf '%s\n' "${role_properties}" | - awk -F '=' -v key="${prop}" ' - BEGIN { - if (key == "login") - key = "canlogin" - else if (key == "superuser") - key = "super" - key = "rol" key - } - $1 == key { - sub(/^[^=]*=/, "") - print - } - ' - ) - - test "${bool_is}" = "${bool_should}" || { - state='different properties' - } - done - - # Check password - passwd_stored=$( - psql_query "SELECT rolpassword FROM pg_authid WHERE rolname = '${rolename}'" \ - | awk 'BEGIN { RS = "\036" } NR == 2 { printf "%s.", $0 }') - passwd_stored=${passwd_stored%.} - - if test -s "${__object:?}/parameter/password" - then - passwd_should=$(cat "${__object:?}/parameter/password"; printf .) - fi - passwd_should=${passwd_should%?.} - - if test -z "${passwd_stored}" - then - test -z "${passwd_should}" || state="${state:-different} password" - elif expr "${passwd_stored}" : 'SCRAM-SHA-256\$.*$' >/dev/null - then - # SCRAM-SHA-256 "encrypted" password - # NOTE: There is currently no easy way to check SCRAM passwords without - # logging in - password_check_login || state="${state:-different} password" - elif expr "${passwd_stored}" : 'md5[0-9a-f]\{32\}$' >/dev/null - then - # MD5 "encrypted" password - if command -v md5sum >/dev/null 2>&1 - then - should_md5=$( - printf '%s%s' "${passwd_should}" "${rolename}" \ - | md5sum - | sed -e 's/[^0-9a-f]*$//') - elif command -v gmd5sum >/dev/null 2>&1 - then - should_md5=$( - printf '%s%s' "${passwd_should}" "${rolename}" \ - | gmd5sum - | sed -e 's/[^0-9a-f]*$//') - elif command -v openssl >/dev/null 2>&1 - then - should_md5=$( - printf '%s%s' "${passwd_should}" "${rolename}" \ - | openssl dgst -md5 | sed 's/^.* //') - fi - - if test -n "${should_md5}" - then - test "${passwd_stored}" = "md5${should_md5}" \ - || state="${state:-different} password" - else - password_check_login || state="${state:-different} password" - fi - else - # unencrypted password (unsupported since PostgreSQL 10) - test "${passwd_stored}" = "${passwd_should}" \ - || state="${state:-different} password" - fi - - test -n "${state}" || state='present' + echo 'present' else - state='absent' + echo 'absent' fi - -echo "${state}" diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 4cb78330..282294c9 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -12,104 +11,55 @@ # # cdist is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -quote() { - if test $# -gt 0 - then - printf '%s' "$*" - else - cat - - fi | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" -} - -postgres_user=$(cat "${__object:?}/explorer/postgres_user") -rolename=${__object_id:?} -state_is=$(cat "${__object:?}/explorer/state") -state_should=$(cat "${__object:?}/parameter/state") - -if test "${state_is}" = "${state_should}" -then - exit 0 -fi - -psql_query() { - printf 'su -l %s -c %s\n' \ - "$(quote "${postgres_user}")" \ - "$(quote "psql postgres -q -w -c $(quote "$1")")" -} - -psql_set_password() { - # NOTE: Always make sure that the password does not end up in psql_history! - # NOTE: Never set an empty string as the password, because it can be - # interpreted differently by different tooling. - if test -s "${__object:?}/parameter/password" - then - cat <<-EOF - exec 3< "\${__object:?}/parameter/password" - su -l '${postgres_user}' -c 'psql -q -w postgres' <<'SQL' - \set HISTFILE /dev/null - \set pw \`cat <&3\` - ALTER ROLE "${rolename}" WITH PASSWORD :'pw'; - SQL - exec 3<&- - EOF - else - psql_query "ALTER ROLE \"${rolename}\" WITH PASSWORD NULL;" - fi -} - -role_properties_should() { - _props= - for _prop in login createdb createrole superuser - do - _props="${_props}${_props:+ }$( - if test -f "${__object:?}/parameter/${_prop}" - then - echo "${_prop}" - else - echo "no${_prop}" - fi \ - | tr '[:lower:]' '[:upper:]')" - done - printf '%s\n' "${_props}" - unset _prop _props -} - -case ${state_should} +case "$(cat "${__global}/explorer/os")" in - (present) - case ${state_is} - in - (absent) - psql_query "CREATE ROLE \"${rolename}\" WITH $(role_properties_should);" - psql_set_password - ;; - (different*) - if expr "${state_is}" : 'different.*properties' >/dev/null - then - psql_query "ALTER ROLE \"${rolename}\" WITH $(role_properties_should);" - fi - - if expr "${state_is}" : 'different.*password' >/dev/null - then - psql_set_password - fi - ;; - (*) - printf 'Invalid state reported by state explorer: %s\n' "${state_is}" >&2 - exit 1 - ;; - esac - ;; - (absent) - printf 'su -l %s -c %s\n' \ - "$(quote "${postgres_user}")" \ - "$(quote "dropuser $(quote "${rolename}")")" - ;; + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + +name="$__object_id" +state_is="$(cat "$__object/explorer/state")" +state_should="$(cat "$__object/parameter/state")" + +[ "$state_is" = "$state_should" ] && exit 0 + +case "$state_should" in + present) + if [ -f "$__object/parameter/password" ]; then + password="$(cat "$__object/parameter/password")" + fi + booleans="" + for boolean in login createdb createrole superuser; do + if [ ! -f "$__object/parameter/$boolean" ]; then + boolean="no${boolean}" + fi + upper=$(echo $boolean | tr '[:lower:]' '[:upper:]') + booleans="$booleans $upper" + done + + [ -n "$password" ] && password="PASSWORD '$password'" + cat << EOF +su - '$postgres_user' -c "psql postgres -wc \"CREATE ROLE \\\\\"$name\\\\\" WITH $password $booleans;\"" +EOF + ;; + absent) + cat << EOF +su - '$postgres_user' -c "dropuser \"$name\"" +EOF + ;; esac diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index 922ce3df..a655bda7 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -1,24 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}" +destination="/$__object_id" -# shellcheck disable=SC2012 -group_gid=$(ls -ldn "${destination}" | awk '{ print $4 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((group_gid + 1)) -ge 0 -then - group_should=$(cat "${__object:?}/parameter/group") - - if expr "${group_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${group_gid}" - else - if command -v getent >/dev/null 2>&1 - then - getent group "${group_gid}" | cut -d : -f 1 - else - awk -F: -v gid="${group_gid}" '$3 == gid { print $1 }' /etc/group - fi - fi -fi +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index ebec751f..8b3c7f8e 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -1,19 +1,5 @@ -#!/bin/sh -e +#!/bin/sh -destination="/${__object_id:?}" +destination="/$__object_id" -# shellcheck disable=SC2012 -owner_uid=$(ls -ldn "${destination}" | awk '{ print $3 }') - -# NOTE: +1 because $((notanum)) prints 0. -if test $((owner_uid + 1)) -ge 0 -then - owner_should=$(cat "${__object:?}/parameter/owner") - - if expr "${owner_should}" : '[0-9]*$' >/dev/null - then - printf '%u\n' "${owner_uid}" - else - printf '%s\n' "$(id -u -n "${owner_uid}")" - fi -fi +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/man.rst b/cdist/conf/type/__pyvenv/man.rst index e2e4a1e6..8085ff12 100644 --- a/cdist/conf/type/__pyvenv/man.rst +++ b/cdist/conf/type/__pyvenv/man.rst @@ -61,7 +61,7 @@ EXAMPLES __pyvenv /home/foo/fooenv --pyvenv /usr/local/bin/pyvenv-3.4 # Create python virtualenv for user foo. - __pyvenv /home/foo/fooenv --group foo --owner foo + __pyvenv /home/foo/fooenv --group foo --user foo # Create python virtualenv with specific parameters. __pyvenv /home/services/djangoenv --venvparams "--copies --system-site-packages" diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local index e9f3c131..e36ded2f 100755 --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,104 +1,39 @@ #!/bin/sh -e - -if ! command -v rsync > /dev/null -then - echo 'rsync is missing in local machine' >&2 - exit 1 -fi - -src="$( cat "$__object/parameter/source" )" - -if [ ! -e "$src" ] -then - echo "$src not found" >&2 - exit 1 -fi - -if [ -f "$__object/parameter/destination" ] -then - dst="$( cat "$__object/parameter/destination" )" -else - dst="/$__object_id" -fi - -# if source is directory, then make sure that -# source and destination are ending with slash, -# because this is what you almost always want when -# rsyncing two directories. - -if [ -d "$src" ] -then - if ! echo "$src" | grep -Eq '/$' - then - src="$src/" - fi - - if ! echo "$dst" | grep -Eq '/$' - then - dst="$dst/" - fi -fi - -remote_user="$( cat "$__object/parameter/remote-user" )" - -options="$( cat "$__object/parameter/options" )" - -if [ -f "$__object/parameter/option" ] -then - while read -r l - do - # there's a limitation in argparse: value can't begin with '-'. - # to workaround this, let's prefix opts with '\' in manifest and remove here. - # read more about argparse issue: https://bugs.python.org/issue9334 - - options="$options $( echo "$l" | sed 's/\\//g' )" - done \ - < "$__object/parameter/option" -fi - -if [ -f "$__object/parameter/owner" ] || [ -f "$__object/parameter/group" ] -then - options="$options --chown=" - - if [ -f "$__object/parameter/owner" ] - then - owner="$( cat "$__object/parameter/owner" )" - options="$options$owner" - fi - - if [ -f "$__object/parameter/group" ] - then - group="$( cat "$__object/parameter/group" )" - options="$options:$group" - fi -fi - -if [ -f "$__object/parameter/mode" ] -then - mode="$( cat "$__object/parameter/mode" )" - options="$options --chmod=$mode" -fi - -# IMPORTANT # -# 1. we first dry-run rsync with change summary to find out -# if there are any changes and code generation is needed. -# 2. normally, to get current state or target host, we run -# such operations in type explorers, but that's not -# possible due to how rsync works. -# 3. redirecting output of dry-run to stderr to ease debugging. -# 4. to understand how that cryptic regex works, please -# open rsync manpage and read about --itemize-changes. +# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# -export RSYNC_RSH="$__remote_exec" +source=$(cat "$__object/parameter/source") +remote_user=$(cat "$__object/parameter/remote-user") -# shellcheck disable=SC2086 -if ! rsync --dry-run --itemize-changes $options "$src" "$remote_user@$__target_host:$dst" \ - | grep -E '^(<|>|c|h|\.|\*)[fdL][cstTpogunbax\.\+\?]+\s' >&2 -then - exit 0 +if [ -f "$__object/parameter/destination" ]; then + destination=$(cat "$__object/parameter/destination") +else + destination="/$__object_id" fi -echo "export RSYNC_RSH='$__remote_exec'" +set -- +if [ -f "$__object/parameter/rsync-opts" ]; then + while read -r opts; do + set -- "$@" "--$opts" + done < "$__object/parameter/rsync-opts" +fi -echo "rsync $options $src $remote_user@$__target_host:$dst" +echo rsync -a \ + --no-owner --no-group \ + -q "$@" "${source}/" "${remote_user}@${__target_host}:${destination}" diff --git a/cdist/conf/type/__postgres_extension/explorer/state b/cdist/conf/type/__rsync/gencode-remote old mode 100644 new mode 100755 similarity index 54% rename from cdist/conf/type/__postgres_extension/explorer/state rename to cdist/conf/type/__rsync/gencode-remote index 9d156be7..074246af --- a/cdist/conf/type/__postgres_extension/explorer/state +++ b/cdist/conf/type/__rsync/gencode-remote @@ -1,7 +1,6 @@ #!/bin/sh -e -# -*- mode: sh; indent-tabs-mode: t -*- # -# 2021 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# 2015 Dominique Roux (dominique.roux4 at gmail.com) # # This file is part of cdist. # @@ -18,24 +17,21 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# Prints "present" if the extension is currently installed. -# "absent" otherwise. -quote() { printf '%s\n' "$*" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/"; } - -postgres_user=$("${__type_explorer:?}/postgres_user") - -IFS=: read -r dbname extname < the type may even vanish + +If you are fine with these constraints, please read on. + + +This cdist type allows you to mirror local directories to the +target host using rsync. Rsync will be installed in the manifest of the type. +If group or owner are giveng, a recursive chown will be executed on the +target host. + +A slash will be appended to the source directory so that only the contents +of the directory are taken and not the directory name itself. REQUIRED PARAMETERS ------------------- source - Source directory in local machine. - If source is directory, slash (``/``) will be added to source and destination paths. + Where to take files from OPTIONAL PARAMETERS ------------------- -destination - Destination directory. Defaults to ``$__object_id``. +group + Group to chgrp to. owner - Will be passed to ``rsync`` as ``--chown=OWNER``. - Read ``rsync(1)`` for more details. + User to chown to. -group - Will be passed to ``rsync`` as ``--chown=:GROUP``. - Read ``rsync(1)`` for more details. - -mode - Will be passed to ``rsync`` as ``--chmod=MODE``. - Read ``rsync(1)`` for more details. - -options - Defaults to ``--recursive --links --perms --times``. - Due to `bug in Python's argparse`_, value must be prefixed with ``\``. +destination + Use this as the base destination instead of the object id remote-user - Defaults to ``root``. + Use this user instead of the default "root" for rsync operations. OPTIONAL MULTIPLE PARAMETERS ---------------------------- -option - Pass additional options to ``rsync``. - See ``rsync(1)`` for all possible options. - Due to `bug in Python's argparse`_, value must be prefixed with ``\``. +rsync-opts + Use this option to give rsync options with. + See rsync(1) for available options. + Only "--" options are supported. + Write the options without the beginning "--" + Can be specified multiple times. + + +MESSAGES +-------- +NONE EXAMPLES -------- + .. code-block:: sh - __rsync /var/www/example.com \ - --owner root \ - --group www-data \ - --mode 'D750,F640' \ - --source "$__files/example.com/www" + # You can use any source directory + __rsync /tmp/testdir \ + --source /etc + + # Use source from type + __rsync /etc \ + --source "$__type/files/package" + + # Allow multiple __rsync objects to write to the same dir + __rsync mystuff \ + --destination /usr/local/bin \ + --source "$__type/files/package" + + __rsync otherstuff \ + --destination /usr/local/bin \ + --source "$__type/files/package2" + + # Use rsync option --exclude + __rsync /tmp/testdir \ + --source /etc \ + --rsync-opts exclude=sshd_conf + + # Use rsync with multiple options --exclude --dry-run + __rsync /tmp/testing \ + --source /home/tester \ + --rsync-opts exclude=id_rsa \ + --rsync-opts dry-run + + +SEE ALSO +-------- +:strong:`rsync`\ (1) AUTHORS ------- -Ander Punnar +Nico Schottelius COPYING ------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. +Copyright \(C) 2015 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest index 64fa804e..9bd44c6d 100755 --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,3 +1,21 @@ #!/bin/sh -e +# +# 2015 Dominique Roux (dominique.roux4 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# __package rsync diff --git a/cdist/conf/type/__rsync/parameter/default/options b/cdist/conf/type/__rsync/parameter/default/options deleted file mode 100644 index d967b110..00000000 --- a/cdist/conf/type/__rsync/parameter/default/options +++ /dev/null @@ -1 +0,0 @@ ---recursive --links --perms --times diff --git a/cdist/conf/type/__rsync/parameter/optional b/cdist/conf/type/__rsync/parameter/optional index 833e9bbe..ac2b2390 100644 --- a/cdist/conf/type/__rsync/parameter/optional +++ b/cdist/conf/type/__rsync/parameter/optional @@ -1,6 +1,4 @@ destination -group -mode -options owner +group remote-user diff --git a/cdist/conf/type/__rsync/parameter/optional_multiple b/cdist/conf/type/__rsync/parameter/optional_multiple index 01925a15..fdb7cd88 100644 --- a/cdist/conf/type/__rsync/parameter/optional_multiple +++ b/cdist/conf/type/__rsync/parameter/optional_multiple @@ -1 +1 @@ -option +rsync-opts diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file deleted file mode 100755 index ec3d0fe8..00000000 --- a/cdist/conf/type/__sed/explorer/file +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -if [ ! -e "$file" ] -then - echo "$file does not exist" >&2 - exit 1 -fi - -cat "$file" diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote deleted file mode 100755 index f99c5a88..00000000 --- a/cdist/conf/type/__sed/gencode-remote +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -script="$( cat "$__object/parameter/script" )" - -if [ "$script" = '-' ] -then - script="$( cat "$__object/stdin" )" -fi - -# since stdin is not available in explorer, we pull file from target with explorer - -file_from_target="$__object/explorer/file" - -sed_cmd='sed' - -if [ -f "$__object/parameter/regexp-extended" ] -then - sed_cmd="$sed_cmd -E" -fi - -# do sed dry run, diff result and if no change, then there's nothing to do -# also redirect diff's output to stderr for debugging purposes - -if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 -then - exit 0 -fi - -# we can't use -i, because it's not posix, so we fly with tempfile and cp -# and we use cp because we want to preserve destination file's attributes - -# shellcheck disable=SC2016 -echo 'tmp="$__object/tempfile"' - -echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - -echo "$script" - -echo 'EOF' - -echo "cp \"\$tmp\" '$file'" - -# shellcheck disable=SC2016 -echo 'rm -f "$tmp"' - -echo 'change' >> "$__messages_out" - -if [ -f "$__object/parameter/onchange" ] -then - cat "$__object/parameter/onchange" -fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst deleted file mode 100644 index 86789363..00000000 --- a/cdist/conf/type/__sed/man.rst +++ /dev/null @@ -1,57 +0,0 @@ -cdist-type__sed(7) -================== - -NAME ----- -cdist-type__sed - Transform text files with ``sed`` - - -DESCRIPTION ------------ -Transform text files with ``sed``. - - -REQUIRED MULTIPLE PARAMETERS ----------------------------- -script - ``sed`` script. - If ``-`` then the script is read from ``stdin``. - - -OPTIONAL PARAMETERS -------------------- -file - Path to the file. Defaults to ``$__object_id``. - -onchange - Execute this command if ``sed`` changes file. - - -BOOLEAN PARAMETERS ------------------- -regexp-extended - Use extended regular expressions in the script. - Might not be supported with every ``sed`` version. - - -EXAMPLES --------- - -.. code-block:: sh - - __sed /tmp/foobar --script 's/foo/bar/' - - echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - - - -AUTHORS -------- -Ander Punnar - - -COPYING -------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean deleted file mode 100644 index 1ad75c5d..00000000 --- a/cdist/conf/type/__sed/parameter/boolean +++ /dev/null @@ -1 +0,0 @@ -regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional deleted file mode 100644 index fa86f917..00000000 --- a/cdist/conf/type/__sed/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -file -onchange diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple deleted file mode 100644 index 84f7e31d..00000000 --- a/cdist/conf/type/__sed/parameter/required_multiple +++ /dev/null @@ -1 +0,0 @@ -script diff --git a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group b/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group deleted file mode 100755 index a6cb3dfd..00000000 --- a/cdist/conf/type/__snakeoil_cert/explorer/ssl-cert-group +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/sh -e - -if grep -Eq '^ssl-cert:' /etc/group -then - echo 'present' -else - echo 'absent' -fi diff --git a/cdist/conf/type/__snakeoil_cert/explorer/state b/cdist/conf/type/__snakeoil_cert/explorer/state deleted file mode 100755 index cc5aae0b..00000000 --- a/cdist/conf/type/__snakeoil_cert/explorer/state +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -e - -key_path="$( cat "$__object/parameter/key-path" )" - -if echo "$key_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - key_path="$( printf "$key_path" "$__object_id" )" -fi - -cert_path="$( cat "$__object/parameter/cert-path" )" - -if echo "$cert_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - cert_path="$( printf "$cert_path" "$__object_id" )" -fi - -if [ ! -f "$key_path" ] || [ ! -f "$cert_path" ] -then - echo 'absent' -else - echo 'present' -fi diff --git a/cdist/conf/type/__snakeoil_cert/gencode-remote b/cdist/conf/type/__snakeoil_cert/gencode-remote deleted file mode 100755 index 8ffbfad1..00000000 --- a/cdist/conf/type/__snakeoil_cert/gencode-remote +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/sh -e - -state="$( cat "$__object/explorer/state" )" - -if [ "$state" = 'present' ] -then - exit 0 -fi - -if [ -f "$__object/parameter/common-name" ] -then - common_name="$( cat "$__object/parameter/common-name" )" -else - common_name="$__object_id" -fi - -key_path="$( cat "$__object/parameter/key-path" )" - -if echo "$key_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - key_path="$( printf "$key_path" "$__object_id" )" -fi - -cert_path="$( cat "$__object/parameter/cert-path" )" - -if echo "$cert_path" | grep -Fq '%s' -then - # shellcheck disable=SC2059 - cert_path="$( printf "$cert_path" "$__object_id" )" -fi - -key_type="$( cat "$__object/parameter/key-type" )" - -key_type_arg="$( echo "$key_type" | cut -d : -f 2 )" - -case "$key_type" in - rsa:*) - echo "openssl genrsa -out '$key_path' $key_type_arg" - ;; - ec:*) - echo "openssl ecparam -name $key_type_arg -genkey -noout -out '$key_path'" - ;; -esac - -# shellcheck disable=SC2016 -echo 'csr_path="$( mktemp )"' - -echo "openssl req -new -subj '/CN=$common_name' -key '$key_path' -out \"\$csr_path\"" - -echo "openssl x509 -req -sha256 -days 3650 -in \"\$csr_path\" -signkey '$key_path' -out '$cert_path'" - -# shellcheck disable=SC2016 -echo 'rm -f "$csr_path"' - -if [ "$( cat "$__object/explorer/ssl-cert-group" )" = 'present' ] -then - key_group='ssl-cert' -else - key_group='root' -fi - -echo "chmod 640 '$key_path'" - -echo "chown root '$key_path'" - -echo "chgrp $key_group '$key_path'" - -echo "chmod 644 '$cert_path'" - -echo "chown root '$cert_path'" - -echo "chgrp root '$cert_path'" diff --git a/cdist/conf/type/__snakeoil_cert/man.rst b/cdist/conf/type/__snakeoil_cert/man.rst deleted file mode 100644 index b0b0a2e9..00000000 --- a/cdist/conf/type/__snakeoil_cert/man.rst +++ /dev/null @@ -1,61 +0,0 @@ -cdist-type__snakeoil_cert(7) -============================ - -NAME ----- -cdist-type__snakeoil_cert - Generate self-signed certificate - - -DESCRIPTION ------------ -The purpose of this type is to generate **self-signed** certificate and private key -for **testing purposes**. Certificate will expire in 3650 days. - -Certificate's and key's access bits will be ``644`` and ``640`` respectively. -If target system has ``ssl-cert`` group, then it will be used as key's group. -Use ``require='__snakeoil_cert/...' __file ...`` to override. - - -OPTIONAL PARAMETERS -------------------- -common-name - Defaults to ``$__object_id``. - -key-path - ``%s`` in path will be replaced with ``$__object_id``. - Defaults to ``/etc/ssl/private/%s.pem``. - -key-type - Possible values are ``rsa:$bits`` and ``ec:$name``. - For possible EC names see ``openssl ecparam -list_curves``. - Defaults to ``rsa:2048``. - -cert-path - ``%s`` in path will be replaced with ``$__object_id``. - Defaults to ``/etc/ssl/certs/%s.pem``. - - -EXAMPLES --------- -.. code-block:: sh - - __snakeoil_cert localhost-rsa \ - --common-name localhost \ - --key-type rsa:4096 - - __snakeoil_cert localhost-ec \ - --common-name localhost \ - --key-type ec:prime256v1 - - -AUTHORS -------- -Ander Punnar - - -COPYING -------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path b/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path deleted file mode 100644 index 4bbae089..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/default/cert-path +++ /dev/null @@ -1 +0,0 @@ -/etc/ssl/certs/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path b/cdist/conf/type/__snakeoil_cert/parameter/default/key-path deleted file mode 100644 index 86eb9359..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/default/key-path +++ /dev/null @@ -1 +0,0 @@ -/etc/ssl/private/%s.pem diff --git a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type b/cdist/conf/type/__snakeoil_cert/parameter/default/key-type deleted file mode 100644 index f13f8ada..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/default/key-type +++ /dev/null @@ -1 +0,0 @@ -rsa:2048 diff --git a/cdist/conf/type/__snakeoil_cert/parameter/optional b/cdist/conf/type/__snakeoil_cert/parameter/optional deleted file mode 100644 index 76d08c0a..00000000 --- a/cdist/conf/type/__snakeoil_cert/parameter/optional +++ /dev/null @@ -1,4 +0,0 @@ -common-name -key-path -key-type -cert-path diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index aca0f2b9..ccab0afc 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -25,7 +25,6 @@ type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ] if [ -n "${type_and_key}" ] then file="$(cat "$__object/parameter/file")" - test -e "$file" || exit 0 # get any entries that match the type and key diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index cbffde94..f37aa565 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -37,10 +37,9 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" - grep -v -F -x '$line' '$file' >\$tmpfile fi -cat "\$tmpfile" >"$file" -rm -f "\$tmpfile" +grep -v -F -x '$line' '$file' > \$tmpfile || true +mv -f "\$tmpfile" "$file" DONE } diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index 9694a64b..cec25746 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,7 +1,6 @@ #!/bin/sh -e # shellcheck disable=SC1090 -# shellcheck disable=SC1091 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] diff --git a/cdist/conf/type/__sshd_config/manifest b/cdist/conf/type/__sshd_config/manifest index e37afebb..566bde90 100755 --- a/cdist/conf/type/__sshd_config/manifest +++ b/cdist/conf/type/__sshd_config/manifest @@ -39,14 +39,7 @@ in (freebsd|netbsd|openbsd) # whitelist ;; - (openbmc-phosphor) - # whitelist - # OpenBMC can be configured with dropbear and OpenSSH. - # If dropbear is used, the state explorer will already fail because it - # cannot find the sshd binary. - ;; (*) - : "${__type:?}" # make shellcheck happy printf 'Your operating system (%s) is currently not supported by this type (%s)\n' \ "${os}" "${__type##*/}" >&2 printf 'Please contribute an implementation for it if you can.\n' >&2 diff --git a/cdist/conf/type/__update_alternatives/explorer/alternatives b/cdist/conf/type/__update_alternatives/explorer/alternatives index bb1619a9..34aaca56 100755 --- a/cdist/conf/type/__update_alternatives/explorer/alternatives +++ b/cdist/conf/type/__update_alternatives/explorer/alternatives @@ -1,4 +1,4 @@ #!/bin/sh -e -LC_ALL=C update-alternatives --display "${__object_id:?}" 2>/dev/null \ -| awk -F ' - ' '/priority [0-9]+$/ { print $1 }' +update-alternatives --display "$__object_id" 2>/dev/null \ + | awk -F ' - ' '/priority [0-9]+$/ { print $1 }' diff --git a/cdist/conf/type/__update_alternatives/explorer/link b/cdist/conf/type/__update_alternatives/explorer/link index d1087c75..6519e7c2 100755 --- a/cdist/conf/type/__update_alternatives/explorer/link +++ b/cdist/conf/type/__update_alternatives/explorer/link @@ -18,12 +18,12 @@ for altdir in \ /var/lib/dpkg/alternatives \ /var/lib/alternatives do - if [ ! -f "$altdir/${__object_id:?}" ] + if [ ! -f "$altdir/$__object_id" ] then continue fi - link="$( awk 'NR==2' "$altdir/${__object_id:?}" )" + link="$( awk 'NR==2' "$altdir/$__object_id" )" if [ -n "$link" ] then @@ -31,12 +31,9 @@ do fi done -if [ -z "$link" ] && [ -z "${__cdist_dry_run+dry run}" ] +if [ -z "$link" ] then - # NOTE: ignore error for dry-runs because a package providing the link - # might be managed by another cdist object (which wasn't executed, - # because dry run…). - echo "unable to get link for ${__object_id:?}" >&2 + echo "unable to get link for $__object_id" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_is b/cdist/conf/type/__update_alternatives/explorer/path_is index 5cf4fa4b..fc304d5d 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_is +++ b/cdist/conf/type/__update_alternatives/explorer/path_is @@ -1,15 +1,11 @@ #!/bin/sh -e -path_is=$( - LC_ALL=C update-alternatives --display "${__object_id?}" 2>/dev/null \ - | awk '/link currently points to/ { print $5 }') +path_is="$( update-alternatives --display "$__object_id" 2>/dev/null \ + | awk '/link currently points to/ {print $5}' )" -if [ -z "$path_is" ] && [ -z "${__cdist_dry_run+dry run}" ] +if [ -z "$path_is" ] then - # NOTE: ignore error for dry-runs because a package providing the - # alternative might be managed by another cdist object (which - # wasn't executed, because dry run…). - echo "unable to get current path for ${__object_id:?}" >&2 + echo "unable to get current path for $__object_id" >&2 exit 1 fi diff --git a/cdist/conf/type/__update_alternatives/explorer/path_should_state b/cdist/conf/type/__update_alternatives/explorer/path_should_state index b74a7ee8..59e015c5 100755 --- a/cdist/conf/type/__update_alternatives/explorer/path_should_state +++ b/cdist/conf/type/__update_alternatives/explorer/path_should_state @@ -1,6 +1,6 @@ #!/bin/sh -e -if [ -f "$( cat "${__object:?}/parameter/path" )" ] +if [ -f "$( cat "$__object/parameter/path" )" ] then echo 'present' else diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index e91ea78f..e393cdef 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -18,39 +18,37 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . -path_is="$( cat "${__object:?}/explorer/path_is" )" +path_is="$( cat "$__object/explorer/path_is" )" -path_should="$( cat "${__object:?}/parameter/path" )" +path_should="$( cat "$__object/parameter/path" )" if [ "$path_is" = "$path_should" ] then exit 0 fi -if [ "$( cat "${__object:?}/explorer/path_should_state" )" = 'absent' ] \ - && [ -z "${__cdist_dry_run+dry run}" ] +if [ "$( cat "$__object/explorer/path_should_state" )" = 'absent' ] && [ -z "$__cdist_dry_run" ] then echo "$path_should does not exist in target" >&2 exit 1 fi -name=${__object_id:?} +name="$__object_id" -if ! grep -Fxq "$path_should" "${__object:?}/explorer/alternatives" +alternatives="$( cat "$__object/explorer/alternatives" )" + +if ! echo "$alternatives" | grep -Fxq "$path_should" then - if [ -f "${__object:?}/parameter/install" ] + if [ ! -f "$__object/parameter/install" ] then - link="$( cat "${__object:?}/explorer/link" )" - echo "update-alternatives --install '$link' '$name' '$path_should' 1000" - elif [ -z "${__cdist_dry_run+dry run}" ] - then - # NOTE: ignore error for dry-runs because a package providing the link - # to be installed might be managed by another cdist object (which - # wasn't executed, because dry run…). echo "$path_should is not in $name alternatives." >&2 echo 'Please install missing packages or use --install to add path to alternatives.' >&2 exit 1 fi + + link="$( cat "$__object/explorer/link" )" + + echo "update-alternatives --install '$link' '$name' '$path_should' 1000" fi echo "update-alternatives --set '$name' '$path_should'" diff --git a/cdist/config.py b/cdist/config.py index 638fdf0e..e84f6f84 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -190,7 +190,7 @@ class Config: fd.write(sys.stdin.read()) except (IOError, OSError) as e: raise cdist.Error(("Creating tempfile for stdin data " - "failed: {}").format(e)) + "failed: %s" % e)) args.manifest = initial_manifest_temp_path atexit.register(lambda: os.remove(initial_manifest_temp_path)) @@ -273,15 +273,15 @@ class Config: host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) - log.debug("Base root path for target host \"%s\" is \"%s\"", - host, host_base_path) + log.debug("Base root path for target host \"{}\" is \"{}\"".format( + host, host_base_path)) hostcnt += 1 if args.parallel: pargs = (host, host_tags, host_base_path, hostdir, args, True, configuration) - log.trace("Args for multiprocessing operation for host %s: %s", - host, pargs) + log.trace(("Args for multiprocessing operation " + "for host {}: {}".format(host, pargs))) process_args.append(pargs) else: try: @@ -298,10 +298,10 @@ class Config: except cdist.Error: failed_hosts.append(host) elif args.parallel: - log.trace("Multiprocessing start method is %s", - multiprocessing.get_start_method()) - log.trace("Starting multiprocessing Pool for %d parallel host" - " operation", args.parallel) + log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + log.trace(("Starting multiprocessing Pool for {} " + "parallel host operation".format(args.parallel))) results = mp_pool_run(cls.onehost, process_args, @@ -396,13 +396,16 @@ class Config: remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) - log.debug("remote_exec for host \"%s\": %s", host, remote_exec) - log.debug("remote_copy for host \"%s\": %s", host, remote_copy) + log.debug("remote_exec for host \"{}\": {}".format( + host, remote_exec)) + log.debug("remote_copy for host \"{}\": {}".format( + host, remote_copy)) family = cls._address_family(args) - log.debug("address family: %s", family) + log.debug("address family: {}".format(family)) target_host = cls.resolve_target_addresses(host, family) - log.debug("target_host for host \"%s\": %s", host, target_host) + log.debug("target_host for host \"{}\": {}".format( + host, target_host)) local = cdist.exec.local.Local( target_host=target_host, @@ -417,9 +420,6 @@ class Config: exec_path=sys.argv[0], save_output_streams=args.save_output_streams) - # Make __global state dir available to custom remote scripts. - os.environ['__global'] = local.base_path - remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, @@ -471,8 +471,8 @@ class Config: """Do what is most often done: deploy & cleanup""" start_time = time.time() - self.log.info("Starting %s run", - 'dry' if self.dry_run else 'configuration') + self.log.info("Starting {} run".format( + 'dry' if self.dry_run else 'configuration')) self._init_files_dirs() @@ -490,9 +490,9 @@ class Config: self._remove_files_dirs() self.local.save_cache(start_time) - self.log.info("Finished %s run in %.2f seconds", + self.log.info("Finished {} run in {:.2f} seconds".format( 'dry' if self.dry_run else 'successful', - time.time() - start_time) + time.time() - start_time)) def cleanup(self): self.log.debug("Running cleanup commands") @@ -516,8 +516,8 @@ class Config: self.local.object_path, self.local.type_path, self.local.object_marker_name): if cdist_object.cdist_type.is_install: - self.log.debug("Running in config mode, ignoring install " - "object: %s", cdist_object) + self.log.debug(("Running in config mode, ignoring install " + "object: {0}").format(cdist_object)) else: yield cdist_object @@ -537,7 +537,7 @@ class Config: objects_changed = False for cdist_object in self.object_list(): - if cdist_object.has_requirements_unfinished( + if cdist_object.requirements_unfinished( cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -548,7 +548,7 @@ class Config: self.object_prepare(cdist_object) objects_changed = True - if cdist_object.has_requirements_unfinished( + if cdist_object.requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -562,13 +562,13 @@ class Config: return objects_changed def _iterate_once_parallel(self): - self.log.debug("Iteration in parallel mode in %d jobs", self.jobs) + self.log.debug("Iteration in parallel mode in {} jobs".format( + self.jobs)) objects_changed = False cargo = [] for cdist_object in self.object_list(): - if cdist_object.has_requirements_unfinished( - cdist_object.requirements): + if cdist_object.requirements_unfinished(cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -585,8 +585,8 @@ class Config: self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.trace("Multiprocessing start method is %s", - multiprocessing.get_start_method()) + self.log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) self.log.trace("Multiprocessing cargo: %s", cargo) @@ -600,8 +600,9 @@ class Config: "sequentially")) self.explorer.transfer_type_explorers(cargo_types.pop()) else: - self.log.trace("Starting multiprocessing Pool for %d " - "parallel types explorers transferring", nt) + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel types explorers transferring".format( + nt))) args = [ (ct, ) for ct in cargo_types ] @@ -610,8 +611,8 @@ class Config: self.log.trace(("Multiprocessing for parallel transferring " "types' explorers finished")) - self.log.trace("Starting multiprocessing Pool for %d parallel " - "objects preparation", n) + self.log.trace(("Starting multiprocessing Pool for {} parallel " + "objects preparation".format(n))) args = [ (c, False, ) for c in cargo ] @@ -622,13 +623,12 @@ class Config: del cargo[:] for cdist_object in self.object_list(): - if cdist_object.has_requirements_unfinished( - cdist_object.requirements): + if cdist_object.requirements_unfinished(cdist_object.requirements): """We cannot do anything for this poor object""" continue if cdist_object.state == core.CdistObject.STATE_PREPARED: - if cdist_object.has_requirements_unfinished( + if cdist_object.requirements_unfinished( cdist_object.autorequire): """The previous step created objects we depend on - wait for them @@ -664,10 +664,10 @@ class Config: self.object_run(chunk[0]) objects_changed = True elif chunk: - self.log.trace("Multiprocessing start method is %s", - multiprocessing.get_start_method()) - self.log.trace("Starting multiprocessing Pool for %d " - "parallel object run", n) + self.log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel object run".format(n))) args = [ (c, ) for c in chunk ] @@ -699,22 +699,20 @@ class Config: check for cycles. ''' graph = {} - - def _add_requirements(cdist_object, requirements): + for cdist_object in self.object_list(): obj_name = cdist_object.name if obj_name not in graph: graph[obj_name] = [] - - for requirement in cdist_object.requirements_unfinished( - requirements): - graph[obj_name].append(requirement.name) - - for cdist_object in self.object_list(): if cdist_object.state == cdist_object.STATE_DONE: continue - _add_requirements(cdist_object, cdist_object.requirements) - _add_requirements(cdist_object, cdist_object.autorequire) + for requirement in cdist_object.requirements_unfinished( + cdist_object.requirements): + graph[obj_name].append(requirement.name) + + for requirement in cdist_object.requirements_unfinished( + cdist_object.autorequire): + graph[obj_name].append(requirement.name) return graph_check_cycle(graph) def iterate_until_finished(self): @@ -768,7 +766,7 @@ class Config: raise cdist.UnresolvableRequirementsError( ("The requirements of the following objects could not be " - "resolved:\n{}").format("\n".join(info_string))) + "resolved:\n%s") % ("\n".join(info_string))) def _handle_deprecation(self, cdist_object): cdist_type = cdist_object.cdist_type @@ -793,9 +791,9 @@ class Config: def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) - self.log.verbose("Preparing object %s", cdist_object.name) - self.log.verbose("Running manifest and explorers for %s", - cdist_object.name) + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( + "Running manifest and explorers for " + cdist_object.name) self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: self.manifest.run_type_manifest(cdist_object) @@ -809,13 +807,13 @@ class Config: def object_run(self, cdist_object): """Run gencode and code for an object""" try: - self.log.verbose("Running object %s", cdist_object.name) + self.log.verbose("Running object " + cdist_object.name) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " - "object: {}").format(cdist_object)) + "object: %s"), cdist_object) # Generate - self.log.debug("Generating code for %s", cdist_object.name) + self.log.debug("Generating code for %s" % (cdist_object.name)) cdist_object.code_local = self.code.run_gencode_local(cdist_object) cdist_object.code_remote = self.code.run_gencode_remote( cdist_object) @@ -824,20 +822,20 @@ class Config: # Execute if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Processing %s", cdist_object.name) + self.log.info("Processing %s" % (cdist_object.name)) if not self.dry_run: if cdist_object.code_local: - self.log.trace("Executing local code for %s", - cdist_object.name) + self.log.trace("Executing local code for %s" + % (cdist_object.name)) self.code.run_code_local(cdist_object) if cdist_object.code_remote: - self.log.trace("Executing remote code for %s", - cdist_object.name) + self.log.trace("Executing remote code for %s" + % (cdist_object.name)) self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) # Mark this object as done - self.log.trace("Finishing run of %s", cdist_object.name) + self.log.trace("Finishing run of " + cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index bb3a65bd..51d61e04 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -34,17 +34,17 @@ class IllegalObjectIdError(cdist.Error): self.message = message or 'Illegal object id' def __str__(self): - return '{}: {}'.format(self.message, self.object_id) + return '%s: %s' % (self.message, self.object_id) class MissingObjectIdError(cdist.Error): def __init__(self, type_name): self.type_name = type_name - self.message = ("Type {} requires object id (is not a " - "singleton type)").format(self.type_name) + self.message = ("Type %s requires object id (is not a " + "singleton type)") % self.type_name def __str__(self): - return '{}'.format(self.message) + return '%s' % (self.message) class CdistObject: @@ -142,7 +142,7 @@ class CdistObject: if self.object_marker in self.object_id.split(os.sep): raise IllegalObjectIdError( self.object_id, ('object_id may not contain ' - '\'{}\'').format(self.object_marker)) + '\'%s\'') % self.object_marker) if '//' in self.object_id: raise IllegalObjectIdError( self.object_id, 'object_id may not contain //') @@ -189,7 +189,7 @@ class CdistObject: object_id=object_id) def __repr__(self): - return ''.format(self.name) + return '' % self.name def __eq__(self, other): """define equality as 'name is the same'""" @@ -247,13 +247,6 @@ class CdistObject: lambda obj: os.path.join(obj.absolute_path, 'typeorder')) typeorder_dep = fsproperty.FileListProperty( lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep')) - # objects without parents are objects specified in init manifest - parents = fsproperty.FileListProperty( - lambda obj: os.path.join(obj.absolute_path, 'parents')) - # objects without children are object of types that do not reuse other - # types - children = fsproperty.FileListProperty( - lambda obj: os.path.join(obj.absolute_path, 'children')) def cleanup(self): try: @@ -277,10 +270,10 @@ class CdistObject: os.makedirs(path, exist_ok=allow_overwrite) except EnvironmentError as error: raise cdist.Error(('Error creating directories for cdist object: ' - '{}: {}').format(self, error)) + '%s: %s') % (self, error)) def requirements_unfinished(self, requirements): - """Return unsatisfied requirements""" + """Return state whether requirements are satisfied""" object_list = [] @@ -291,14 +284,3 @@ class CdistObject: object_list.append(cdist_object) return object_list - - def has_requirements_unfinished(self, requirements): - """Return whether requirements are satisfied""" - - for requirement in requirements: - cdist_object = self.object_from_name(requirement) - - if cdist_object.state != self.STATE_DONE: - return True - - return False diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index b68448bc..c0329c8a 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -34,7 +34,7 @@ class InvalidTypeError(cdist.Error): self.source_path = os.path.realpath(self.type_absolute_path) def __str__(self): - return "Invalid type '{}' at '{}' defined at '{}'".format( + return "Invalid type '%s' at '%s' defined at '%s'" % ( self.type_path, self.type_absolute_path, self.source_path) @@ -82,9 +82,9 @@ class CdistType: yield cls(base_path, name) except InvalidTypeError as e: # ignore invalid type, log warning and continue - cls.log.warning("Ignoring invalid type '%s' at '%s' defined" - " at '%s'", e.type_path, e.type_absolute_path, - e.source_path) + msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % ( + e.type_path, e.type_absolute_path, e.source_path) + cls.log.warning(msg) # remove invalid from runtime conf dir os.remove(e.type_absolute_path) @@ -109,7 +109,7 @@ class CdistType: return cls._instances[name] def __repr__(self): - return ''.format(self.name) + return '' % self.name def __eq__(self, other): return isinstance(other, self.__class__) and self.name == other.name diff --git a/cdist/core/code.py b/cdist/core/code.py index 12888ba4..1e9b4f80 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -122,8 +122,8 @@ class Code: def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type - gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which)) - script = os.path.join(self.local.type_path, gencode_attr) + script = os.path.join(self.local.type_path, + getattr(cdist_type, 'gencode_%s_path' % which)) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) @@ -167,8 +167,8 @@ class Code: def _run_code(self, cdist_object, which, env=None): which_exec = getattr(self, which) - code_attr = getattr(cdist_object, 'code_{}_path'.format(which)) - script = os.path.join(which_exec.object_path, code_attr) + script = os.path.join(which_exec.object_path, + getattr(cdist_object, 'code_%s_path' % which)) if which_exec.save_output_streams: stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 48414ade..a3baa959 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -131,17 +131,18 @@ class Explorer: self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.debug("Running global explorers in %s parallel jobs", - self.jobs) - self.log.trace("Multiprocessing start method is %s", - multiprocessing.get_start_method()) - self.log.trace("Starting multiprocessing Pool for global explorers" - " run") + self.log.debug("Running global explorers in {} parallel jobs".format( + self.jobs)) + self.log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.trace(("Starting multiprocessing Pool for global " + "explorers run")) args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.trace("Multiprocessing run for global explorers finished") + self.log.trace(("Multiprocessing run for global explorers " + "finished")) # logger is not pickable, so remove it when we pickle def __getstate__(self): @@ -160,8 +161,8 @@ class Explorer: self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path, self.jobs) - self.remote.run(["chmod", "0700", "{}/*".format( - self.remote.global_explorer_path)]) + self.remote.run(["chmod", "0700", + "%s/*" % (self.remote.global_explorer_path)]) def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" @@ -183,14 +184,15 @@ class Explorer: in the object. """ - self.log.verbose("Running type explorers for %s", - cdist_object.cdist_type) + self.log.verbose("Running type explorers for {}".format( + cdist_object.cdist_type)) if transfer_type_explorers: self.log.trace("Transferring type explorers for type: %s", cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) else: - self.log.trace("No need for transferring type explorers for %s", + self.log.trace(("No need for transferring type explorers for " + "type: %s"), cdist_object.cdist_type) self.log.trace("Transferring object parameters for object: %s", cdist_object.name) @@ -234,15 +236,15 @@ class Explorer: remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.trace("Skipping retransfer of type explorers for: %s", - cdist_type) + self.log.trace(("Skipping retransfer of type explorers " + "for: %s"), cdist_type) else: source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) self.remote.transfer(source, destination) - self.remote.run(["chmod", "0700", "{}/*".format(destination)]) + self.remote.run(["chmod", "0700", "%s/*" % (destination)]) self._type_explorers_transferred.append(cdist_type.name) def transfer_object_parameters(self, cdist_object): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 09e74dac..390340d4 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -80,12 +80,13 @@ class NoInitialManifestError(cdist.Error): if user_supplied: if os.path.islink(manifest_path): - self.message = "{}: {} -> {}".format( - msg_header, manifest_path, os.path.realpath(manifest_path)) + self.message = "%s: %s -> %s" % ( + msg_header, manifest_path, + os.path.realpath(manifest_path)) else: - self.message = "{}: {}".format(msg_header, manifest_path) + self.message = "%s: %s" % (msg_header, manifest_path) else: - self.message = "{}".format(msg_header) + self.message = "%s" % (msg_header) def __str__(self): return repr(self.message) @@ -106,7 +107,7 @@ class Manifest: self._open_logger() self.env = { - 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), + 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__global': self.local.base_path, @@ -160,7 +161,7 @@ class Manifest: raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" - self.log.verbose("Running initial manifest %s", initial_manifest) + self.log.verbose("Running initial manifest " + initial_manifest) which = "init" if self.local.save_output_streams: stderr_path = os.path.join(self.local.stderr_base_path, which) diff --git a/cdist/emulator.py b/cdist/emulator.py index c55b47d2..a2bdc3d4 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -36,8 +36,8 @@ from cdist.core.manifest import Manifest class MissingRequiredEnvironmentVariableError(cdist.Error): def __init__(self, name): self.name = name - self.message = ("Emulator requires the environment variable {} to be " - "setup").format(self.name) + self.message = ("Emulator requires the environment variable %s to be " + "setup" % self.name) def __str__(self): return self.message @@ -106,9 +106,8 @@ class Emulator: self.save_stdin() self.record_requirements() self.record_auto_requirements() - self.record_parent_child_relationships() - self.log.trace("Finished %s %s", self.cdist_object.path, - self.parameters) + self.log.trace("Finished %s %s" % ( + self.cdist_object.path, self.parameters)) def __init_log(self): """Setup logging facility""" @@ -170,7 +169,7 @@ class Emulator: # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.trace('Args: %s', self.args) + self.log.trace('Args: %s' % self.args) def init_object(self): # Initialize object - and ensure it is not in args @@ -231,18 +230,18 @@ class Emulator: if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: obj_params = self._object_params_in_context() if obj_params != self.parameters: - errmsg = ("Object {} already exists with conflicting " - "parameters:\n{}: {}\n{}: {}").format( + errmsg = ("Object %s already exists with conflicting " + "parameters:\n%s: %s\n%s: %s" % ( self.cdist_object.name, " ".join(self.cdist_object.source), obj_params, self.object_source, - self.parameters) + self.parameters)) raise cdist.Error(errmsg) else: if self.cdist_object.exists: - self.log.debug('Object %s override forced with CDIST_OVERRIDE', - self.cdist_object.name) + self.log.debug(('Object %s override forced with ' + 'CDIST_OVERRIDE'), self.cdist_object.name) self.cdist_object.create(True) else: self.cdist_object.create() @@ -260,8 +259,8 @@ class Emulator: parent = self.cdist_object.object_from_name(__object_name) parent.typeorder.append(self.cdist_object.name) if self._order_dep_on(): - self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s', - depname, parent.name) + self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' + ' for %s'), depname, parent.name) parent.typeorder_dep.append(depname) elif self._order_dep_on(): self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', @@ -292,7 +291,7 @@ class Emulator: fd.write(chunk) chunk = self._read_stdin() except EnvironmentError as e: - raise cdist.Error('Failed to read from stdin: {}'.format(e)) + raise cdist.Error('Failed to read from stdin: %s' % e) def record_requirement(self, requirement): """record requirement and return recorded requirement""" @@ -301,14 +300,16 @@ class Emulator: try: cdist_object = self.cdist_object.object_from_name(requirement) except core.cdist_type.InvalidTypeError as e: - self.log.error("%s requires object %s, but type %s does not" - " exist. Defined at %s", self.cdist_object.name, - requirement, e.name, self.object_source) + self.log.error(("%s requires object %s, but type %s does not" + " exist. Defined at %s" % ( + self.cdist_object.name, + requirement, e.name, self.object_source))) raise except core.cdist_object.MissingObjectIdError: - self.log.error("%s requires object %s without object id." - " Defined at %s", self.cdist_object.name, - requirement, self.object_source) + self.log.error(("%s requires object %s without object id." + " Defined at %s" % (self.cdist_object.name, + requirement, + self.object_source))) raise self.log.debug("Recording requirement %s for %s", @@ -378,9 +379,10 @@ class Emulator: self.env['require'] += " " + lastcreatedtype else: self.env['require'] = lastcreatedtype - self.log.debug("Injecting require for" - " CDIST_ORDER_DEPENDENCY: %s for %s", - lastcreatedtype, self.cdist_object.name) + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, + self.cdist_object.name) except IndexError: # if no second last line, we are on the first type, # so do not set a requirement @@ -388,7 +390,7 @@ class Emulator: if "require" in self.env: requirements = self.env['require'] - self.log.debug("reqs = %s", requirements) + self.log.debug("reqs = " + requirements) for requirement in self._parse_require(requirements): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: @@ -418,21 +420,3 @@ class Emulator: self.log.debug("Recording autorequirement %s for %s", current_object.name, parent.name) parent.autorequire.append(current_object.name) - - def record_parent_child_relationships(self): - # __object_name is the name of the object whose type manifest is - # currently executed - __object_name = self.env.get('__object_name', None) - if __object_name: - # The object whose type manifest is currently run - parent = self.cdist_object.object_from_name(__object_name) - # The object currently being defined - current_object = self.cdist_object - if current_object.name not in parent.children: - self.log.debug("Recording child %s for %s", - current_object.name, parent.name) - parent.children.append(current_object.name) - if parent.name not in current_object.parents: - self.log.debug("Recording parent %s for %s", - parent.name, current_object.name) - current_object.parents.append(parent.name) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 370336ad..e0aab190 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -152,10 +152,10 @@ class Local: def _setup_object_marker_file(self): with open(self.object_marker_file, 'w') as fd: - fd.write("{}\n".format(self.object_marker_name)) + fd.write("%s\n" % self.object_marker_name) - self.log.trace("Object marker %s saved in %s", - self.object_marker_name, self.object_marker_file) + self.log.trace("Object marker %s saved in %s" % ( + self.object_marker_name, self.object_marker_file)) def _init_cache_dir(self, cache_dir): home_dir = cdist.home_dir() @@ -184,7 +184,7 @@ class Local: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: {}".format(command)) + "list or tuple argument expected, got: %s" % command) quiet = self.quiet_mode or quiet_mode do_save_output = save_output and not quiet and self.save_output_streams @@ -289,12 +289,14 @@ class Local: return cache_subpath def save_cache(self, start_time=time.time()): - self.log.trace("cache subpath pattern: %s", self.cache_path_pattern) + self.log.trace("cache subpath pattern: {}".format( + self.cache_path_pattern)) cache_subpath = self._cache_subpath(start_time, self.cache_path_pattern) - self.log.debug("cache subpath: %s", cache_subpath) + self.log.debug("cache subpath: {}".format(cache_subpath)) destination = os.path.join(self.cache_path, cache_subpath) - self.log.trace("Saving cache %s to %s", self.base_path, destination) + self.log.trace(("Saving cache: " + self.base_path + " to " + + destination)) if not os.path.exists(destination): shutil.move(self.base_path, destination) @@ -330,7 +332,7 @@ class Local: # Iterate over all directories and link the to the output dir for conf_dir in self.conf_dirs: - self.log.debug("Checking conf_dir %s ...", conf_dir) + self.log.debug("Checking conf_dir %s ..." % (conf_dir)) for sub_dir in CONF_SUBDIRS_LINKED: current_dir = os.path.join(conf_dir, sub_dir) @@ -348,13 +350,12 @@ class Local: if os.path.exists(dst): os.unlink(dst) - self.log.trace("Linking %s to %s ...", src, dst) + self.log.trace("Linking %s to %s ..." % (src, dst)) try: os.symlink(src, dst) except OSError as e: - raise cdist.Error( - "Linking {} {} to {} failed: {}".format( - sub_dir, src, dst, e.__str__())) + raise cdist.Error("Linking %s %s to %s failed: %s" % ( + sub_dir, src, dst, e.__str__())) def _link_types_for_emulator(self): """Link emulator to types""" @@ -367,5 +368,5 @@ class Local: os.symlink(src, dst) except OSError as e: raise cdist.Error( - "Linking emulator from {} to {} failed: {}".format( + "Linking emulator from %s to %s failed: %s" % ( src, dst, e.__str__())) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index af9e70cb..e5af2f34 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -24,10 +24,12 @@ import os import glob import subprocess import logging +import multiprocessing import cdist import cdist.exec.util as util import cdist.util.ipaddr as ipaddr +from cdist.mputil import mp_pool_run def _wrap_addr(addr): @@ -174,19 +176,19 @@ class Remote: # create archive tarpath, fcnt = autil.tar(source, self.archiving_mode) if tarpath is None: - self.log.trace("Files count %d is lower than %d limit, " - "skipping archiving", - fcnt, autil.FILES_LIMIT) + self.log.trace(("Files count {} is lower than {} limit, " + "skipping archiving").format( + fcnt, autil.FILES_LIMIT)) else: - self.log.trace("Archiving mode, tarpath: %s, file count: " - "%s", tarpath, fcnt) + self.log.trace(("Archiving mode, tarpath: %s, file count: " + "%s"), tarpath, fcnt) # get archive name tarname = os.path.basename(tarpath) self.log.trace("Archiving mode tarname: %s", tarname) # archive path at the remote desttarpath = os.path.join(destination, tarname) - self.log.trace("Archiving mode desttarpath: %s", - desttarpath) + self.log.trace( + "Archiving mode desttarpath: %s", desttarpath) # transfer archive to the remote side self.log.trace("Archiving mode: transferring") self._transfer_file(tarpath, desttarpath) @@ -260,10 +262,9 @@ class Remote: # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g. # /bin/csh will execute this script in the right way. if env: - remote_env = [" export {env[0]}={env[1]};".format(env=item) - for item in env.items()] - string_cmd = ("/bin/sh -c '{}{}'").format(" ".join(remote_env), - " ".join(command)) + remote_env = [" export %s=%s;" % item for item in env.items()] + string_cmd = ("/bin/sh -c '" + " ".join(remote_env) + + " ".join(command) + "'") cmd.append(string_cmd) else: cmd.extend(command) @@ -277,7 +278,7 @@ class Remote: """ assert isinstance(command, (list, tuple)), ( - "list or tuple argument expected, got: {}".format(command)) + "list or tuple argument expected, got: %s" % command) close_stdout = False close_stderr = False diff --git a/cdist/install.py b/cdist/install.py index d077baef..7c894fe5 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -47,4 +47,4 @@ class Install(cdist.config.Config): yield cdist_object else: self.log.debug("Running in install mode, ignoring non install" - "object: %s", cdist_object) + "object: {0}".format(cdist_object)) diff --git a/cdist/integration.py b/cdist/integration.py index 04470ea7..17b65f09 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.abc.Iterable): + elif isinstance(host, collections.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/inventory.py b/cdist/inventory.py index 106052a2..0387f326 100644 --- a/cdist/inventory.py +++ b/cdist/inventory.py @@ -92,7 +92,7 @@ class Inventory: self.init_db() def init_db(self): - self.log.trace("Init db: %s", self.db_basedir) + self.log.trace("Init db: {}".format(self.db_basedir)) if not os.path.exists(self.db_basedir): os.makedirs(self.db_basedir, exist_ok=True) elif not os.path.isdir(self.db_basedir): @@ -182,9 +182,9 @@ class Inventory: configuration = cfg.get_config(section='GLOBAL') determine_default_inventory_dir(args, configuration) - log.debug("Using inventory: %s", args.inventory_dir) - log.trace("Inventory args: %s", vars(args)) - log.trace("Inventory command: %s", args.subcommand) + log.debug("Using inventory: {}".format(args.inventory_dir)) + log.trace("Inventory args: {}".format(vars(args))) + log.trace("Inventory command: {}".format(args.subcommand)) if args.subcommand == "list": c = InventoryList(hosts=args.host, istag=args.tag, @@ -237,16 +237,16 @@ class InventoryList(Inventory): def _do_list(self, it_tags, it_hosts, check_func): if (it_tags is not None): param_tags = set(it_tags) - self.log.trace("param_tags: %s", param_tags) + self.log.trace("param_tags: {}".format(param_tags)) else: param_tags = set() for host in it_hosts: - self.log.trace("host: %s", host) + self.log.trace("host: {}".format(host)) tags = self._get_host_tags(host) if tags is None: - self.log.debug("Host \'%s\' not found, skipped", host) + self.log.debug("Host \'{}\' not found, skipped".format(host)) continue - self.log.trace("tags: %s", tags) + self.log.trace("tags: {}".format(tags)) if check_func(tags, param_tags): yield host, tags @@ -308,11 +308,11 @@ class InventoryHost(Inventory): def _action(self, host): if self.action == "add": - self.log.debug("Adding host \'%s\'", host) + self.log.debug("Adding host \'{}\'".format(host)) elif self.action == "del": - self.log.debug("Deleting host \'%s\'", host) + self.log.debug("Deleting host \'{}\'".format(host)) hostpath = self._host_path(host) - self.log.trace("hostpath: %s", hostpath) + self.log.trace("hostpath: {}".format(hostpath)) if self.action == "add" and not os.path.exists(hostpath): self._new_hostpath(hostpath) else: @@ -372,23 +372,23 @@ class InventoryTag(Inventory): print("Host \'{}\' does not exist, skipping".format(host), file=sys.stderr) return - self.log.trace("existing host_tags: %s", host_tags) + self.log.trace("existing host_tags: {}".format(host_tags)) if self.action == "del" and self.all: host_tags = set() else: for tag in self.input_tags: if self.action == "add": - self.log.debug("Adding tag \'%s\' for host \'%s\'", - tag, host) + self.log.debug("Adding tag \'{}\' for host \'{}\'".format( + tag, host)) host_tags.add(tag) elif self.action == "del": - self.log.debug("Deleting tag \'%s\' for host \'%s\'", - tag, host) + self.log.debug("Deleting tag \'{}\' for host " + "\'{}\'".format(tag, host)) if tag in host_tags: host_tags.remove(tag) - self.log.trace("new host tags: %s", host_tags) + self.log.trace("new host tags: {}".format(host_tags)) if not self._write_host_tags(host, host_tags): - self.log.trace("%s does not exist, skipped", host) + self.log.trace("{} does not exist, skipped".format(host)) def run(self): if self.allhosts: diff --git a/cdist/log.py b/cdist/log.py index 62e457fe..113f3b4c 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -36,27 +36,25 @@ import threading logging.OFF = logging.CRITICAL + 10 # disable logging logging.addLevelName(logging.OFF, 'OFF') - logging.VERBOSE = logging.INFO - 5 logging.addLevelName(logging.VERBOSE, 'VERBOSE') -def _verbose(self, msg, *args, **kwargs): - self.log(logging.VERBOSE, msg, args, **kwargs) +def _verbose(msg, *args, **kwargs): + logging.log(logging.VERBOSE, msg, *args, **kwargs) -logging.Logger.verbose = _verbose - +logging.verbose = _verbose logging.TRACE = logging.DEBUG - 5 logging.addLevelName(logging.TRACE, 'TRACE') -def _trace(self, msg, *args, **kwargs): - self.log(logging.TRACE, msg, *args, **kwargs) +def _trace(msg, *args, **kwargs): + logging.log(logging.TRACE, msg, *args, **kwargs) -logging.Logger.trace = _trace +logging.trace = _trace class CdistFormatter(logging.Formatter): diff --git a/cdist/message.py b/cdist/message.py index 0c4e21a6..ffa8c2bb 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -70,7 +70,7 @@ class Message: with open(self.global_messages, 'a') as fd: for line in content: - fd.write("{}:{}".format(self.prefix, line)) + fd.write("%s:%s" % (self.prefix, line)) def merge_messages(self): self._merge_messages() diff --git a/cdist/preos.py b/cdist/preos.py index 45711a41..f8a5dd67 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -49,7 +49,7 @@ def scan_preos_dir_plugins(dir): c = cm[1] yield from preos_plugin(c) except ImportError as e: - log.warning("Cannot import '%s': %s", module_name, e) + log.warning("Cannot import '{}': {}".format(module_name, e)) def find_preos_plugins(): @@ -102,7 +102,7 @@ class PreOS: parser.add_argument('remainder_args', nargs=argparse.REMAINDER) args = parser.parse_args(argv[1:]) cdist.argparse.handle_loglevel(args) - log.debug("preos args : %s", args) + log.debug("preos args : {}".format(args)) conf_dirs = util.resolve_conf_dirs_from_config_and_args(args) @@ -122,7 +122,7 @@ class PreOS: func_args = [preos, args.remainder_args, ] else: func_args = [args.remainder_args, ] - log.info("Running preos : %s", preos_name) + log.info("Running preos : {}".format(preos_name)) func(*func_args) else: raise cdist.Error( diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index f1f750ee..a20cdb9c 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -166,7 +166,7 @@ class Debian: args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir) cdist.argparse.handle_loglevel(args) - log.debug("preos: %s, args: %s", cls._preos_name, args) + log.debug("preos: {}, args: {}".format(cls._preos_name, args)) try: env = vars(args) new_env = {} @@ -190,30 +190,27 @@ class Debian: env = new_env env.update(os.environ) cls.update_env(env) - log.debug("preos: %s env: %s", cls._preos_name, env) - - if log.getEffectiveLevel() <= logging.INFO: - info_msg = ["Running preos: {}, suite: {}, arch: {}".format( - cls._preos_name, args.suite, args.arch), ] - if args.mirror: - info_msg.append("mirror: {}".format(args.mirror)) - if args.script: - info_msg.append("script: {}".format(args.script)) - if args.bootstrap: - info_msg.append("bootstrapping") - if args.configure: - info_msg.append("configuring") - if args.pxe_boot_dir: - info_msg.append("creating PXE") - if args.drive: - info_msg.append("creating bootable drive") - log.info(info_msg) - + log.debug("preos: {} env: {}".format(cls._preos_name, env)) cmd = os.path.join(cls._files_dir, "code") - log.debug("cmd=%s", cmd) + info_msg = ["Running preos: {}, suite: {}, arch: {}".format( + cls._preos_name, args.suite, args.arch), ] + if args.mirror: + info_msg.append("mirror: {}".format(args.mirror)) + if args.script: + info_msg.append("script: {}".format(args.script)) + if args.bootstrap: + info_msg.append("bootstrapping") + if args.configure: + info_msg.append("configuring") + if args.pxe_boot_dir: + info_msg.append("creating PXE") + if args.drive: + info_msg.append("creating bootable drive") + log.info(info_msg) + log.debug("cmd={}".format(cmd)) subprocess.check_call(cmd, env=env, shell=True) except subprocess.CalledProcessError as e: - log.error("preos %s failed: %s", cls._preos_name, e) + log.error("preos {} failed: {}".format(cls._preos_name, e)) class Ubuntu(Debian): diff --git a/cdist/scan/commandline.py b/cdist/scan/commandline.py index ddbe4933..eca4cf13 100644 --- a/cdist/scan/commandline.py +++ b/cdist/scan/commandline.py @@ -20,98 +20,36 @@ # import logging -import sys -from datetime import datetime log = logging.getLogger("scan") -def run(scan, args): - # We run each component in a separate process since they - # must not block on each other. +# define this outside of the class to not handle scapy import errors by default +def commandline(args): + log.debug(args) + + try: + import cdist.scan.scan as scan + except ModuleNotFoundError: + print('cdist scan requires scapy to be installed') + processes = [] + if not args.mode: + # By default scan and trigger, but do not call any action + args.mode = ['scan', 'trigger', ] + if 'trigger' in args.mode: - t = scan.Trigger(interfaces=args.interface, - sleeptime=args.trigger_delay) + t = scan.Trigger(interfaces=args.interfaces) t.start() processes.append(t) log.debug("Trigger started") if 'scan' in args.mode: - s = scan.Scanner( - autoconfigure='config' in args.mode, - interfaces=args.interface, - name_mapper=args.name_mapper) + s = scan.Scanner(interfaces=args.interfaces, args=args) s.start() processes.append(s) log.debug("Scanner started") for process in processes: process.join() - - -def list(scan, args): - s = scan.Scanner(interfaces=args.interface, name_mapper=args.name_mapper) - hosts = s.list() - - # A full IPv6 addresses id composed of 8 blocks of 4 hexa chars + - # 6 colons. - ipv6_max_size = 8 * 4 + 10 - date_max_size = len(datetime.now().strftime(scan.datetime_format)) - name_max_size = 25 - - print("{} | {} | {} | {}".format( - 'name'.ljust(name_max_size), - 'address'.ljust(ipv6_max_size), - 'last seen'.ljust(date_max_size), - 'last configured'.ljust(date_max_size))) - print('=' * (name_max_size + 3 + ipv6_max_size + 2 * (3 + date_max_size))) - for host in hosts: - last_seen = host.last_seen() - if last_seen: - last_seen = last_seen.strftime(scan.datetime_format) - else: - last_seen = '-' - - last_configured = host.last_configured() - if last_configured is not None: - last_configured = last_configured.strftime(scan.datetime_format) - else: - last_configured = '-' - - print("{} | {} | {} | {}".format( - host.name(default='-').ljust(name_max_size), - host.address().ljust(ipv6_max_size), - last_seen.ljust(date_max_size), - last_configured.ljust(date_max_size))) - - -# CLI processing is defined outside of the main scan class to handle -# non-available optional scapy dependency (instead of crashing mid-flight). -def commandline(args): - log.debug(args) - - # Check if we have the optional scapy dependency available. - try: - import cdist.scan.scan as scan - except ModuleNotFoundError: - log.error('cdist scan requires scapy to be installed. Exiting.') - sys.exit(1) - - # Set default operation mode. - if not args.mode: - # By default scan and trigger, but do not call any action. - args.mode = ['scan', 'trigger', ] - - if 'config' in args.mode and args.name_mapper is None: - print('--name-mapper must be specified for scanner config mode.', - file=sys.stderr) - sys.exit(1) - - # Print known hosts and exit is --list is specified - do not start - # the scanner. - if args.list: - list(scan, args) - else: - run(scan, args) diff --git a/cdist/scan/scan.py b/cdist/scan/scan.py index 4a20f511..b1d0e9e1 100644 --- a/cdist/scan/scan.py +++ b/cdist/scan/scan.py @@ -19,6 +19,38 @@ # # +# +# Interface to be implemented: +# - cdist scan --mode {scan, trigger, install, config}, --mode can be repeated +# scan: scan / listen for icmp6 replies +# trigger: send trigger to multicast +# config: configure newly detected hosts +# install: install newly detected hosts +# +# Scanner logic +# - save results to configdir: +# basedir = ~/.cdist/scan/ +# last_seen = ~/.cdist/scan//last_seen -- record unix time +# or similar +# last_configured = ~/.cdist/scan//last_configured -- record +# unix time or similar +# last_installed = ~/.cdist/scan//last_configured -- record +# unix time or similar +# +# +# +# +# cdist scan --list +# Show all known hosts including last seen flag +# +# Logic for reconfiguration: +# +# - record when configured last time +# - introduce a parameter --reconfigure-after that takes time argument +# - reconfigure if a) host alive and b) reconfigure-after time passed +# + + from multiprocessing import Process import os import logging @@ -29,84 +61,7 @@ import datetime import cdist.config -logging.basicConfig(level=logging.DEBUG) log = logging.getLogger("scan") -datetime_format = '%Y-%m-%d %H:%M:%S' - - -class Host(object): - def __init__(self, addr, outdir, name_mapper=None): - self.addr = addr - self.workdir = os.path.join(outdir, addr) - self.name_mapper = name_mapper - - os.makedirs(self.workdir, exist_ok=True) - - def __get(self, key, default=None): - fname = os.path.join(self.workdir, key) - value = default - if os.path.isfile(fname): - with open(fname, "r") as fd: - value = fd.readline() - return value - - def __set(self, key, value): - fname = os.path.join(self.workdir, key) - with open(fname, "w") as fd: - fd.write(f"{value}") - - def name(self, default=None): - if self.name_mapper is None: - return default - - fpath = os.path.join(os.getcwd(), self.name_mapper) - if os.path.isfile(fpath) and os.access(fpath, os.X_OK): - out = subprocess.run([fpath, self.addr], capture_output=True) - if out.returncode != 0: - return default - else: - value = out.stdout.decode() - return (default if len(value) == 0 else value) - else: - return default - - def address(self): - return self.addr - - def last_seen(self, default=None): - raw = self.__get('last_seen') - if raw: - return datetime.datetime.strptime(raw, datetime_format) - else: - return default - - def last_configured(self, default=None): - raw = self.__get('last_configured') - if raw: - return datetime.datetime.strptime(raw, datetime_format) - else: - return default - - def seen(self): - now = datetime.datetime.now().strftime(datetime_format) - self.__set('last_seen', now) - - # XXX: There's no easy way to use the config module without feeding it with - # CLI args. Might as well call everything from scratch! - def configure(self): - target = self.name() or self.address() - cmd = ['cdist', 'config', '-v', target] - - fname = os.path.join(self.workdir, 'last_configuration_log') - with open(fname, "w") as fd: - log.debug("Executing: %s", cmd) - completed_process = subprocess.run(cmd, stdout=fd, stderr=fd) - if completed_process.returncode != 0: - log.error("%s return with non-zero code %i - see %s for \ - details.", cmd, completed_process.returncode, fname) - - now = datetime.datetime.now().strftime(datetime_format) - self.__set('last_configured', now) class Trigger(object): @@ -114,14 +69,12 @@ class Trigger(object): Trigger an ICMPv6EchoReply from all hosts that are alive """ - def __init__(self, interfaces, sleeptime, verbose=False): + def __init__(self, interfaces=None, verbose=False): self.interfaces = interfaces - - # Used by scapy / send in trigger/2. self.verbose = verbose - # Delay in seconds between sent ICMPv6EchoRequests. - self.sleeptime = sleeptime + # Wait 5 seconds before triggering again - FIXME: add parameter + self.sleeptime = 5 def start(self): self.processes = [] @@ -140,14 +93,9 @@ class Trigger(object): time.sleep(self.sleeptime) def trigger(self, interface): - try: - log.debug("Sending ICMPv6EchoRequest on %s", interface) - packet = IPv6( - dst="ff02::1%{}".format(interface) - ) / ICMPv6EchoRequest() - send(packet, verbose=self.verbose) - except Exception as e: - log.error("Could not send ICMPv6EchoRequest: %s", e) + packet = IPv6(dst=f"ff02::1%{interface}") / ICMPv6EchoRequest() + log.debug(f"Sending request on {interface}") + send(packet, verbose=self.verbose) class Scanner(object): @@ -155,62 +103,41 @@ class Scanner(object): Scan for replies of hosts, maintain the up-to-date database """ - def __init__(self, interfaces, autoconfigure=False, outdir=None, - name_mapper=None): + def __init__(self, interfaces=None, args=None, outdir=None): self.interfaces = interfaces - self.autoconfigure = autoconfigure - self.name_mapper = name_mapper - self.config_delay = datetime.timedelta(seconds=3600) if outdir: self.outdir = outdir else: self.outdir = os.path.join(os.environ['HOME'], '.cdist', 'scan') - os.makedirs(self.outdir, exist_ok=True) - - self.running_configs = {} def handle_pkg(self, pkg): if ICMPv6EchoReply in pkg: - host = Host(pkg['IPv6'].src, self.outdir, self.name_mapper) - if host.name(): - log.verbose("Host %s (%s) is alive", host.name(), - host.address()) - else: - log.verbose("Host %s is alive", host.address()) + host = pkg['IPv6'].src + log.verbose(f"Host {host} is alive") - host.seen() + dir = os.path.join(self.outdir, host) + fname = os.path.join(dir, "last_seen") - # Configure if needed. - if self.autoconfigure and \ - host.last_configured(default=datetime.datetime.min) + \ - self.config_delay < datetime.datetime.now(): - self.config(host) + now = datetime.datetime.now() - def list(self): - hosts = [] - for addr in os.listdir(self.outdir): - hosts.append(Host(addr, self.outdir, self.name_mapper)) + os.makedirs(dir, exist_ok=True) - return hosts + # FIXME: maybe adjust the format so we can easily parse again + with open(fname, "w") as fd: + fd.write(f"{now}\n") - def config(self, host): - if host.name() is None: - log.debug("config - could not resolve name for %s, aborting.", - host.address()) - return + def config(self): + """ + Configure a host - previous_config_process = self.running_configs.get(host.name()) - if previous_config_process is not None and \ - previous_config_process.is_alive(): - log.debug("config - is already running for %s, aborting.", - host.name()) + - Assume we are only called if necessary + - However we need to ensure to not run in parallel + - Maybe keep dict storing per host processes + - Save the result + - Save the output -> probably aligned to config mode - log.info("config - running against host %s (%s).", host.name(), - host.address()) - p = Process(target=host.configure()) - p.start() - self.running_configs[host.name()] = p + """ def start(self): self.process = Process(target=self.scan) @@ -221,9 +148,47 @@ class Scanner(object): def scan(self): log.debug("Scanning - zzzzz") - try: - sniff(iface=self.interfaces, - filter="icmp6", - prn=self.handle_pkg) - except Exception as e: - log.error("Could not start listener: %s", e) + sniff(iface=self.interfaces, + filter="icmp6", + prn=self.handle_pkg) + + +if __name__ == '__main__': + t = Trigger(interfaces=["wlan0"]) + t.start() + + # Scanner can listen on many interfaces at the same time + s = Scanner(interfaces=["wlan0"]) + s.scan() + + # Join back the trigger processes + t.join() + + # Test in my lan shows: + # [18:48] bridge:cdist% ls -1d fe80::* + # fe80::142d:f0a5:725b:1103 + # fe80::20d:b9ff:fe49:ac11 + # fe80::20d:b9ff:fe4c:547d + # fe80::219:d2ff:feb2:2e12 + # fe80::21b:fcff:feee:f446 + # fe80::21b:fcff:feee:f45c + # fe80::21b:fcff:feee:f4b1 + # fe80::21b:fcff:feee:f4ba + # fe80::21b:fcff:feee:f4bc + # fe80::21b:fcff:feee:f4c1 + # fe80::21d:72ff:fe86:46b + # fe80::42b0:34ff:fe6f:f6f0 + # fe80::42b0:34ff:fe6f:f863 + # fe80::42b0:34ff:fe6f:f9b2 + # fe80::4a5d:60ff:fea1:e55f + # fe80::77a3:5e3f:82cc:f2e5 + # fe80::9e93:4eff:fe6c:c1f4 + # fe80::ba69:f4ff:fec5:6041 + # fe80::ba69:f4ff:fec5:8db7 + # fe80::bad8:12ff:fe65:313d + # fe80::bad8:12ff:fe65:d9b1 + # fe80::ce2d:e0ff:fed4:2611 + # fe80::ce32:e5ff:fe79:7ea7 + # fe80::d66d:6dff:fe33:e00 + # fe80::e2ff:f7ff:fe00:20e6 + # fe80::f29f:c2ff:fe7c:275e diff --git a/cdist/shell.py b/cdist/shell.py index 05803556..04a68937 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -65,7 +65,7 @@ class Shell: def _init_environment(self): self.env = os.environ.copy() additional_env = { - 'PATH': "{}:{}".format(self.local.bin_path, os.environ['PATH']), + 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), # for use in type emulator '__cdist_type_base_path': self.local.type_path, '__cdist_manifest': "cdist shell", diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index 11619e96..a9c20cd3 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -86,7 +86,8 @@ class ObjectClassTestCase(test.CdistTestCase): def test_create_singleton(self): """Check whether creating an object without id (singleton) works""" - self.expected_objects[0].object_from_name("__test_singleton") + singleton = self.expected_objects[0].object_from_name( + "__test_singleton") # came here - everything fine def test_create_singleton_not_singleton_type(self): @@ -125,16 +126,16 @@ class ObjectIdTestCase(test.CdistTestCase): def test_object_id_contains_object_marker(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = 'object_id/may/not/contain/{}/anywhere'.format( - OBJECT_MARKER_NAME) + illegal_object_id = ( + 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME) with self.assertRaises(core.IllegalObjectIdError): core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker_string(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = 'object_id/may/contain_{}_in_filename'.format( - OBJECT_MARKER_NAME) + illegal_object_id = ( + 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME) core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) # if we get here, the test passed @@ -194,32 +195,28 @@ class ObjectTestCase(test.CdistTestCase): def test_path(self): self.assertEqual(self.cdist_object.path, - "__third/moon/{}".format(OBJECT_MARKER_NAME)) + "__third/moon/%s" % OBJECT_MARKER_NAME) def test_absolute_path(self): self.assertEqual(self.cdist_object.absolute_path, os.path.join(self.object_base_path, - "__third/moon/{}".format( - OBJECT_MARKER_NAME))) + "__third/moon/%s" % OBJECT_MARKER_NAME)) def test_code_local_path(self): self.assertEqual(self.cdist_object.code_local_path, - "__third/moon/{}/code-local".format( - OBJECT_MARKER_NAME)) + "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) def test_code_remote_path(self): self.assertEqual(self.cdist_object.code_remote_path, - "__third/moon/{}/code-remote".format( - OBJECT_MARKER_NAME)) + "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) def test_parameter_path(self): self.assertEqual(self.cdist_object.parameter_path, - "__third/moon/{}/parameter".format( - OBJECT_MARKER_NAME)) + "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) def test_explorer_path(self): self.assertEqual(self.cdist_object.explorer_path, - "__third/moon/{}/explorer".format(OBJECT_MARKER_NAME)) + "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) def test_parameters(self): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 4b2bc3ba..befd7b57 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -84,8 +84,8 @@ class EmulatorTestCase(test.CdistTestCase): def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] - self.env['require'] = "__file/bad/id/with/{}/inside".format( - self.local.object_marker_name) + self.env['require'] = ( + "__file/bad/id/with/%s/inside") % self.local.object_marker_name emu = emulator.Emulator(argv, env=self.env) self.assertRaises(core.IllegalObjectIdError, emu.run) diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index b23f8447..a7fe384d 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -47,8 +47,8 @@ class RemoteTestCase(test.CdistTestCase): args = (self.target_host,) kwargs.setdefault('base_path', self.base_path) user = getpass.getuser() - kwargs.setdefault('remote_exec', 'ssh -o User={} -q'.format(user)) - kwargs.setdefault('remote_copy', 'scp -o User={} -q'.format(user)) + kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user) + kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user) if 'stdout_base_path' not in kwargs: stdout_path = os.path.join(self.temp_dir, 'stdout') os.makedirs(stdout_path, exist_ok=True) @@ -170,7 +170,7 @@ class RemoteTestCase(test.CdistTestCase): r = self.create_remote(remote_exec=remote_exec, remote_copy=remote_copy) self.assertEqual(r.run('true', return_output=True), - "{}\n".format(self.target_host[0])) + "%s\n" % self.target_host[0]) def test_run_script_target_host_in_env(self): handle, remote_exec_path = self.mkstemp(dir=self.temp_dir) @@ -185,7 +185,7 @@ class RemoteTestCase(test.CdistTestCase): with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "true"]) self.assertEqual(r.run_script(script, return_output=True), - "{}\n".format(self.target_host[0])) + "%s\n" % self.target_host[0]) def test_run_script_with_env_target_host_in_env(self): handle, script = self.mkstemp(dir=self.temp_dir) diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py index 55040fc9..61cd5d97 100644 --- a/cdist/test/message/__init__.py +++ b/cdist/test/message/__init__.py @@ -67,7 +67,7 @@ class MessageTestCase(test.CdistTestCase): def test_message_merge_prefix(self): """Ensure messages are merged and are prefixed""" - expectedcontent = "{}:{}".format(self.prefix, self.content) + expectedcontent = "%s:%s" % (self.prefix, self.content) out = self.message.env['__messages_out'] diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 6bf935e8..1d76fd76 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -30,10 +30,10 @@ class AbsolutePathRequiredError(cdist.Error): self.path = path def __str__(self): - return 'Absolute path required, got: {}'.format(self.path) + return 'Absolute path required, got: %s' % self.path -class FileList(collections.abc.MutableSequence): +class FileList(collections.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.abc.MutableSequence): self.__write(lines) -class DirectoryDict(collections.abc.MutableMapping): +class DirectoryDict(collections.MutableMapping): """A dict that stores it's items as files in a directory. """ @@ -218,7 +218,7 @@ class FileBasedProperty: def _get_attribute(self, instance, owner): name = self._get_property_name(owner) - attribute_name = '__{}'.format(name) + attribute_name = '__%s' % name if not hasattr(instance, attribute_name): path = self._get_path(instance) attribute_instance = self.attribute_class(path) diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index d9e5f498..95ca74ee 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -42,7 +42,8 @@ def resolve_target_host_name(host, family=0): # gethostbyaddr returns triple # (hostname, aliaslist, ipaddrlist) host_name = socket.gethostbyaddr(ip_addr)[0] - log.debug("derived host_name for host \"%s\": %s", host, host_name) + log.debug("derived host_name for host \"{}\": {}".format( + host, host_name)) except (socket.gaierror, socket.herror) as e: # in case of error provide empty value host_name = '' @@ -53,7 +54,8 @@ def resolve_target_fqdn(host): log = logging.getLogger(host) try: host_fqdn = socket.getfqdn(host) - log.debug("derived host_fqdn for host \"%s\": %s", host, host_fqdn) + log.debug("derived host_fqdn for host \"{}\": {}".format( + host, host_fqdn)) except socket.herror as e: # in case of error provide empty value host_fqdn = '' diff --git a/docs/changelog b/docs/changelog index 00defc2a..b2b35616 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,69 +1,9 @@ Changelog --------- -7.0.0: 2022-07-31 - * Explorer machine_type: Rewrite (Dennis Camera) - * New type: __sed (Ander Punnar) - * New type: __haproxy_dualstack (Evilham and ungleich) - * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) - * Type __apt_source: Fix complaint about suite change (Matthias Stecher) - * Type __package_apt: Fix complaint about suite change (Matthias Stecher) - * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) - * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) - * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) - * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) - * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) - * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) - * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) - * Type __apt_source: Add signed-by parameter (Daniel Fancsali) - * Explorer: add support for checkpoint (Stephan Leemburg) - -6.9.8: 2021-08-24 - * Type __rsync: Rewrite (Ander Punnar) - * New type: __apt_pin (Daniel Fancsali) - * Explorer os_version: Convert Devuan ceres to version number (Dennis Camera) - * Core: Fix logging bug (Dennis Camera) - * Build: Improve Makefile compatibility (Evilham) - * Type __filesystem: Support ubuntu (Joachim Desroches) - * Explorer os_version: Fall back to os-release/lsb-release file on Ubuntu (Dennis Camera) - * Explorer memory: Fix conversion of large numbers (>= 2GiB) (Dennis Camera) - * Type __update_alternatives: Fix dry run and non-English systems (Dennis Camera) - * Explorer os_version: Fix for FreeBSD < 10.0 and for legacy Mac OS X versions (Dennis Camera) - * Explorer os_version: Add bookworm and trixie debian code names, fallback to 99.99 for unknown code name in sid (Ander Punnar) - -6.9.7: 2021-07-10 - * New type: __postgres_conf (Beni Ruef, Dennis Camera) - * Types __postgres_*: Improve OS support and do some cleanup (Dennis Camera) - * Type __apt_key_uri: Deprecate in favour of __apt_key --uri (Evilham) - * Type __apt_key: Documentation improvements, support in-type/in-manifest provision with --source, make fallback to apt-key(8) explicit with --use-deprecated-apt-key (Evilham) - * Type __letsencrypt_cert: Bugfix, performance; revamp explorers, add locking (Evilham) - * Type __git: Fix group explorer (Ander Punnar) - * Type __pyvenv: Fix group explorer (Dennis Camera) - * Type __download: Improve checksum verification, add optional --destination (Ander Punnar) - * Type __debconf_set_selections: Add state explorer (Dennis Camera) - * Core: Implement usable cdist scan (Timothée Floure) - * New type: __snakeoil_cert (Ander Punnar) - * Type __rsync: Honour $__remote_exec env var (Daniel Fancsali) - -6.9.6: 2021-04-20 - * Type __pyvenv: Fix user example in man page (Dennis Camera) - * Core: config: Make local state directory available to custom remotes (Steven Armstrong - * Type __ssh_authorized_key: grep only if file exists (Dennis Camera) - * Type __sshd_config: Whitelist OpenBMC (Dennis Camera) - * Core: Maintain object relationship graph in cdist cache (Darko Poljak) - * Type __git: Fix numeric owner and group handling (Dennis Camera) - * Type __pyvenv: Fix numeric owner and group handling (Dennis Camera) - * Type __download: Make sum parameter optional (Ander Punnar) - -6.9.5: 2021-02-28 +next: * Core: preos: Fix passing cdist debug parameter (Darko Poljak) * Type __sshd_config: Produce error if invalid config is generated, fix processing of AuthenticationMethods and AuthorizedKeysFile, document explorer bug (Dennis Camera) - * Explorer memory: Fix result units; support Solaris (Dennis Camera) - * Type __postgres_role: Implement modification of roles (Dennis Camera) - * Type __letsencrypt_cert: Fix issues with hooks (Evil Ham) - * Type __package_pip: Add optional extra dependencies param (Matthias Stecher) 6.9.4: 2020-12-21 * Type __package_pkgng_freebsd: Fix bootstrapping pkg (Dennis Camera) @@ -174,7 +114,7 @@ Changelog * Type __pf_ruleset: Refactor (Kamila Součková, Evil Ham) * Type __pf_apply: Deprecate type (Kamila Součková, Evil Ham) * Configuration: Add notes to cdist.cfg.skeleton (Evil Ham) - * Explorers cpu_cores, memory: Improve BSD support (Evil Ham) + * Explorers cpu_cores, memory: Improve *BSD support (Evil Ham) * Core: Remove debug logging noise (Evil Ham) 6.5.4: 2020-04-11 @@ -239,7 +179,7 @@ Changelog * Documentation: PreOS english nitpicking (Evil Ham) * Documentation: Add installing from source with signature verification (Darko Poljak) * Core: preos: Support top command logging options, custom conf-dir option and CDIST_PATH env var (Darko Poljak) - * Type __start_on_boot: Docs: remove unsupported BSD claim (Evil Ham) + * Type __start_on_boot: Docs: remove unsupported *BSD claim (Evil Ham) * New type: __openldap_server (Evil Ham) 6.2.0: 2019-11-30 @@ -1098,9 +1038,9 @@ Changelog * Removed type __removeline (replaced by __line) (Nico Schottelius) * Type __directory: Parameter --parents and --recursive are now boolean (Nico Schottelius) * Type __package_apt, __package_luarocks, __package_opkg, - __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, - __package_rubygem, __package_yum, __process: - Parameter state accepts only "present" and "absent" (Nico Schottelius) + __package_pacman, __package_pkg_freebsd, __package_pkg_openbsd, + __package_rubygem, __package_yum, __process: + Parameter state accepts only "present" and "absent" (Nico Schottelius) * Dist: Initial support for pypi packaging (Nico Schottelius) 2.0.15: 2012-11-02 diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org deleted file mode 100644 index 42b4f5c5..00000000 --- a/docs/dev/release-process.org +++ /dev/null @@ -1,90 +0,0 @@ -* Install requirements (Alpine) - - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ - py3-build twine -* Ensure your gpg setup works with the email used in the git commit! - - For me this is nico@nico-notebook.schottelius.org - - Signature / id is on nb2 -* Create ~/.pypirc -[distutils] - index-servers = - pypi - cdist - -[pypi] - username = __token__ - password = ... - -[cdist] - repository = https://upload.pypi.org/legacy/ - username = __token__ - password = ... - -* Add date in docs/changelog -* Run ./bin/cdist-build-helper -* TODO Move to "build" - - python3 -m build -* DONE git tag: when? -CLOSED: [2022-07-31 Sun 23:58] -** Asked during release process: ok -* DONE Pypi error with distutils: do not use distutils anymore -CLOSED: [2022-07-31 Sun 23:58] -python3 setup.py sdist upload -... -Creating tar archive -removing 'cdist-7.0.0' (and everything under it) -running upload -Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ -Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. -error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. -(venv2) [22:50] nb2:cdist% - -* DONE Pypi error with twine: fixed in twine 4.0.1 -CLOSED: [2022-07-31 Sun 23:58] - -Seeing: - -(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* -Uploading distributions to https://upload.pypi.org/legacy/ -Traceback (most recent call last): - File "/usr/bin/twine", line 8, in - sys.exit(main()) - File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main - result = cli.dispatch(sys.argv[1:]) - File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch - return main(args.args) - File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main - return upload(upload_settings, parsed_args.dists) - File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload - resp = repository.upload(package) - File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload - resp = self._upload(package) - File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload - data = package.metadata_dictionary() - File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary - "dynamic": meta.dynamic, -AttributeError: 'Wheel' object has no attribute 'dynamic' - - -Fix: - - -(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* -⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. -Uploading distributions to https://upload.pypi.org/legacy/ -Uploading cdist-7.0.0-py3-none-any.whl -100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s -Uploading cdist-7.0.0.tar.gz -100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s - -View at: -https://pypi.org/project/cdist/7.0.0/ -* TODO cdist web - - on staticweb-2022 - - Should be moved to sftp/k8s - - - Manual steps: - - ~/bin/permissions.public html/ - rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ - ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest" diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst index d4159e77..d2d2d56c 100644 --- a/docs/src/cdist-cache.rst +++ b/docs/src/cdist-cache.rst @@ -61,14 +61,6 @@ Object cache overview ~~~~~~~~~~~~~~~~~~~~~ Each object under :strong:`object` directory has its own structure. -autorequire - file containing a list of object auto requirements - -children - file containing a list of object children, i.e. objects of types that this - type reuses (along with 'parents' it is used for maintaining parent-child - relationship graph) - code-local code generated from gencode-local, present only if something is generated @@ -88,15 +80,6 @@ parameter directory containing type parameter named files containing parameter values -parents - file containing a list of object parents, i.e. objects of types that reuse - this type (along with 'children' it is used for maintaining parent-child - relationship graph); objects without parents are objects specified in init - manifest - -require - file containing a list of object requirements - source this type's source (init manifest) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 390ab9ec..18863145 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -12,7 +12,7 @@ This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.5 * SSH client - * sphinx with the rtd theme (for building html docs and/or the man pages) + * sphinx (for building html docs and/or the man pages) Target Hosts ~~~~~~~~~~~~ diff --git a/docs/src/cdist-scan.rst b/docs/src/cdist-scan.rst deleted file mode 100644 index 86b7fab6..00000000 --- a/docs/src/cdist-scan.rst +++ /dev/null @@ -1,84 +0,0 @@ -Scan -===== - -Description ------------ -Runs cdist as a daemon that discover/watch on hosts and reconfigure them -periodically. It is especially useful in netboot-based environment where hosts -boot unconfigured, and to ensure your infrastructure stays in sync with your -configuration. - -This feature is still consider to be in **beta** stage, and only operate on -IPv6 (including link-local). - -Usage (Examples) ----------------- - -Discover hosts on local network and configure those whose name is resolved by -the name mapper script. - -.. code-block:: sh - - $ cdist scan --beta --interface eth0 \ - --mode scan --name-mapper path/to/script \ - --mode trigger --mode config - -List known hosts and exit. - -.. code-block:: sh - - $ cdist scan --beta --list --name-mapper path/to/script - -Please refer to `cdist(1)` for a detailed list of parameters. - -Modes ------ - -The scanner has 3 modes that can be independently toggled. If the `--mode` -parameter is not specified, only `tigger` and `scan` are enabled (= hosts are -not configured). - -trigger - Send ICMPv6 requests to specific hosts or broadcast over IPv6 link-local to - trigger detection by the `scan` module. - -scan - Watch for incoming ICMPv6 replies and optionally configure detected hosts. - -config - Enable configuration of hosts detected by `scan`. - -Name Mapper Script ------------------- - -The name mapper script takes an IPv6 address as first argument and writes the -resolved name to stdout - if any. The script must be executable. - -Simplest script: - -.. code-block:: sh - - #!/bin/sh - - case "$1" in - "fe80::20d:b9ff:fe57:3524") - printf "my-host-01" - ;; - "fe80::7603:bdff:fe05:89bb") - printf "my-host-02" - ;; - esac - -Resolving name from `PTR` DNS record: - -.. code-block:: sh - - #!/bin/sh - - for cmd in dig sed; do - if ! command -v $cmd > /dev/null; then - exit 1 - fi - done - - dig +short -x "$1" | sed -e 's/.$//' diff --git a/docs/src/conf.py b/docs/src/conf.py index a3dfafca..47765413 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -copyright = 'ungleich GmbH 2021' +copyright = 'ungleich GmbH 2020' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/src/index.rst b/docs/src/index.rst index 369d5309..31c044dc 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -34,7 +34,6 @@ It natively supports IPv6 since the first release. cdist-parallelization cdist-inventory cdist-preos - cdist-scan cdist-integration cdist-reference cdist-best-practice diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 599ec3b7..0ecb4a61 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -88,9 +88,6 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] - cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] - [-d CONFIG_DELAY] [-t TRIGGER_DELAY] - DESCRIPTION ----------- @@ -644,31 +641,6 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. -SCAN ----- - -Runs cdist as a daemon that discover/watch on hosts and reconfigure them -periodically. - -**-I INTERFACE, --interfaces INTERFACE** - Interface to listen on. Can be specified multiple times. - -**-m MODE, --mode MODE** - Scanner components to enable. Can be specified multiple time to enable more - than one component. Supported modes are: scan, trigger and config. Defaults - to tiggger and scan. - -**--name-mapper PATH_TO_SCRIPT** - Path to script used to resolve a remote host name from an IPv6 address. - -**--list** - List known hosts and exit. - -**-d CONFIG_DELAY, --config-delay CONFIG_DELAY** - How long (seconds) to wait before reconfiguring after last try (config mode only). - -**-t TRIGGER_DELAY, --tigger-delay TRIGGER_DELAY** - How long (seconds) to wait between ICMPv6 echo requests (trigger mode only). CONFIGURATION -------------