diff --git a/bin/cdist-build-helper b/bin/cdist-build-helper index 6f514ef5..0380b3f8 100755 --- a/bin/cdist-build-helper +++ b/bin/cdist-build-helper @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011-2022 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. @@ -534,8 +534,7 @@ eof ;; version) - target_version="$(git describe | sed 's/-/.dev/; s/-/+/g')" - printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py + printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py ;; target-version) diff --git a/cdist/__init__.py b/cdist/__init__.py index 31d49889..b1903136 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -63,6 +63,9 @@ REMOTE_COPY = "scp -o User=root -q" REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" +ORDER_DEP_STATE_NAME = 'order_dep_state' +TYPEORDER_DEP_NAME = 'typeorder_dep' + MIN_SUPPORTED_PYTHON_VERSION = (3, 5) diff --git a/cdist/argparse.py b/cdist/argparse.py index 8f7bbb85..9c24ae25 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -5,6 +5,7 @@ import logging import collections import functools import cdist.configuration +import cdist.trigger import cdist.log import cdist.preos import cdist.info @@ -12,7 +13,7 @@ import cdist.scan.commandline # set of beta sub-commands -BETA_COMMANDS = set(('install', 'inventory', 'scan', )) +BETA_COMMANDS = set(('install', 'inventory', 'scan', 'trigger', )) # set of beta arguments for sub-commands BETA_ARGS = { 'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )), @@ -472,6 +473,9 @@ def get_parsers(): parser['info'].set_defaults(func=cdist.info.Info.commandline) # Scan = config + further + parser['scan'] = parser['sub'].add_parser('scan', add_help=False, + parents=[parser['config']]) + parser['scan'] = parser['sub'].add_parser( 'scan', parents=[parser['loglevel'], parser['beta'], @@ -509,6 +513,28 @@ def get_parsers(): help='How long (seconds) to wait between ICMPv6 echo requests') parser['scan'].set_defaults(func=cdist.scan.commandline.commandline) + # Trigger + parser['trigger'] = parser['sub'].add_parser( + 'trigger', parents=[parser['loglevel'], + parser['beta'], + parser['common'], + parser['config_main']]) + parser['trigger'].add_argument( + '-D', '--directory', action='store', required=False, + help=('Where to create local files')) + parser['trigger'].add_argument( + '-H', '--http-port', action='store', default=3000, required=False, + help=('Create trigger listener via http on specified port')) + parser['trigger'].add_argument( + '--ipv6', default=False, + help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), + action='store_true') + parser['trigger'].add_argument( + '-O', '--source', action='store', required=False, + help=('Which file to copy for creation')) + + parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline) + for p in parser: parser[p].epilog = EPILOG diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index c9fb5cdf..26bb8e3d 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - awk '{printf("%s\n", $(NF-1))}' /etc/cp-release - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index 7279a9c2..b1009627 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - cat /etc/cp-release - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 1f91cc40..82ff9977 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - echo "CheckPoint" - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 0bb9f7fe..5ebfff1a 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -21,9 +21,6 @@ set +e case "$("$__explorer/os")" in - checkpoint) - sed /etc/cp-release -e 's/.* R\([1-9][0-9]*\)\.[0-9]*$/\1/' - ;; openwrt) # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index c31f5ca6..1c84f4d7 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -1,6 +1,8 @@ -#!/bin/sh -e +#!/bin/sh # -# 2021 Dennis Camera (cdist at dtnr.ch) +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# 2020 Evilham (contact at evilham.com) # # This file is part of cdist. # @@ -17,1019 +19,91 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# This explorer tries to determine what type of machine the target to be -# configured is (container, virtual machine, bare-metal). -# -# It will print one line for each layer it can detect. -# The format of all lines is: TYPE[ VERB VENDOR] -# -# VERB does not have a special meaning, it is just for better readability. -# -# e.g. -# container -# container on lxc -# virtual by kvm-spapr -# -# The third word of each line (except the first) can be composed of different -# parts concatenated with a `-' (minus) character, with each component being -# a specification of the previous, e.g.: -# - lxc-libvirt (LXC container, managed by libvirt) -# - lpar-s390 / lpar-power (LPAR running on IBM S/390 or POWER, respectively) -# - xen-hvm / xen-pv (Xen HVM vs para-virtualization) -# -# If this explorer cannot collect enough information about virtualization it -# will fall back to 'physical'. -# -# Add /sbin and /usr/sbin to the path so we can find system -# binaries like dmidecode. -PATH=$(getconf PATH 2>/dev/null) || PATH='/usr/bin:/bin' -PATH="/sbin:/usr/sbin:${PATH}" -export PATH +os=$("$__explorer/os") -arch=$(uname -m | sed -e 's/i.86/i386/' -e 's/arm.*/arm/') -uname_s=$(uname -s) - - -is_command() { command -v "$1" >/dev/null 2>&1; } - -files_same() { - # shellcheck disable=SC2012 - LC_ALL=C df -P "$1" "$2" 2>/dev/null | { - read -r _ # skip header line - read -r fs1 _ _ _ _ mp1 - read -r fs2 _ _ _ _ mp2 - test "${fs1}" = "${fs2}" -a "${mp1}" = "${mp2}" || return 1 - } && - ls -1Ldi "$1" "$2" 2>/dev/null | { - read -r ino1 _ - read -r ino2 _ - test "${ino1}" = "${ino2}" || return 1 - } +vendor_string_to_machine_type() { + for vendor in vmware bochs kvm qemu virtualbox bhyve; do + if echo "${1}" | grep -q -i "${vendor}"; then + if [ "${vendor}" = "bochs" ] || [ "${vendor}" = "qemu" ]; then + vendor="kvm" + fi + echo "virtual_by_${vendor}" + exit + fi + done } -is_oneof() ( - x=$1; shift - for y - do - test "${x}" = "${y}" || continue - return 0 - done - return 1 -) - -tolower() { LC_ALL=C tr '[:upper:]' '[:lower:]'; } - -# shellcheck disable=SC2086 -glob_exists() { set -- $1; test -e "$1"; } - -get_dmi_field() { - if is_oneof "${uname_s}" NetBSD - then - case $1 - in - (system-manufacturer) _mib=machdep.dmi.system-vendor ;; - (system-product-name) _mib=machdep.dmi.system-product ;; - (system-version|system-uuid) _mib=machdep.dmi.$1 ;; - (bios-vendor|bios-version) _mib=machdep.dmi.$1 ;; - (biod-release-date) _mib=machdep.dmi.bios-date ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - if is_command dmidecode - then - dmidecode -s "$1" - elif test -d "${dmi_sysfs-}" - then - case $1 - in - (system-manufacturer) _filename=sys_vendor ;; - (system-product-name) _filename=product_name ;; - (*) _filename=$(echo "$1" | tr - _) ;; - esac - if test -r "${dmi_sysfs-}/${_filename}" - then - cat "${dmi_sysfs}/${_filename}" - fi - unset _filename - elif test "${uname_s}" = OpenBSD - then - # NOTE: something similar to system-manufacutrer and system-product-name - # is available on OpenBSD in sysctl - case $1 - in - (system-manufacturer) _mib=hw.vendor ;; - (system-product-name) _mib=hw.product ;; - (*) _mib= ;; - esac - - test -n "${_mib}" && get_sysctl "${_mib}" | grep -e . && return - fi - - return 1 -} - -has_cpuinfo() { test -e /proc/cpuinfo; } - -get_sysctl() { - is_command sysctl && sysctl -n "$1" 2>/dev/null -} - -detected_layer() { - test -n "${_toplayer:-}" || echo "${_toplayer:=${1:?}}" -} - - -# Check for chroot - -has_chroot_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -r' -} - -check_chroot_systemd() { - systemd-detect-virt -r -} - -has_chroot_debian_ischroot() { - is_command ischroot -} - -check_chroot_debian_ischroot() { - ischroot --default-false -} - -has_chroot_procfs() { - test -d /proc/ -} - -check_chroot_procfs() ( - is_chroot=false # default - if test -e /proc/1/root && ! files_same /proc/1/root / - then - is_chroot=true - fi - if test -e /proc/1/mountinfo -a -e /proc/self/mountinfo - then - has_mountinfo=true - cmp -s /proc/1/mountinfo /proc/self/mountinfo || is_chroot=true - fi - - if ${is_chroot} - then - # try to determine where the chroot has been mounted - rootdev=$(LC_ALL=C df -P / | awk 'NR==2{print $1}') - - if test -e "${rootdev}" - then - # escape chroot to determine where the device containing the - # chroot's / is mounted - rootdevmnt=$(LC_ALL=C chroot /proc/1/root df -P "${rootdev}" | awk 'NR==2{print $6}') - - # shellcheck disable=SC2012 - root_ino=$(ls -1di / | awk '{print $1}') - - # escape chroot and find mount point by inode - chroot /proc/1/root find "${rootdevmnt}" -xdev -type d -inum "${root_ino}" - elif ${has_mountinfo} - then - while read -r mntid _ _ _ cmntpnt _ - do - read -r _ _ _ _ hmntpnt _ <<-EOF - $(grep -e "^$((mntid)) " /proc/1/mountinfo) - EOF - printf '%s\n' "${hmntpnt%${cmntpnt}}" - done /dev/null) && - case ${_ctengine} - in - (''|'none') - return 1 ;; - ('container-other') - return 0 ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - echo "${_ctengine}" ;; - esac -) - -has_ct_pid_1() { - test -r /run/systemd/container -o -r /proc/1/environ -} - -translate_container_name() { - case $1 - in - ('lxc') - echo lxc ;; - ('lxc-libvirt') - echo lxc-libvirt ;; - ('podman') - echo podman ;; - ('systemd-nspawn') - echo systemd_nspawn ;; - (*) - return 1 ;; - esac - return 0 -} - -check_ct_pid_1() { - if test -r /run/systemd/container - then - translate_container_name "$(head -n1 /run/systemd/container)" \ - && return 0 - fi - - if test -r /proc/1/environ - then - translate_container_name "$( - LC_ALL=C tr '\000' '\n' /dev/null - then - # https://github.com/Microsoft/WSL/issues/423#issuecomment-221627364 - echo wsl - elif test -d /var/.cagefs - then - # https://docs.cloudlinux.com/cloudlinux_os_components/#cagefs - # CageFS is not "really" a container, but it isn't a chroot either. - echo cagefs - elif test -e /proc/self/status && grep -q -e '^VxID: [0-9]\{1,\}' /proc/self/status - then - # Linux-VServer - if grep -q -x -F 'VxID: 0' /proc/self/status - then - # host - return 1 - else - # guest - echo linux_vserver - fi - else - return 1 - fi -} - -check_ct_os_specific() ( - if jailed=$(get_sysctl security.jail.jailed) && test "${jailed}" = 1 - then - # FreeBSD jail - echo jail - return 0 - fi - - if is_command zonename && test "$(zonename)" != global - then - # Solaris zone - echo zone - return 0 - fi - - return 1 -) - - -# Check for hypervisor - -guess_hypervisor_from_cpu_model() { - case $1 - in - (*\ KVM\ *) - echo kvm ;; - (*\ QEMU\ *|QEMU\ *) - echo qemu ;; - (*) - return 1 ;; - esac -} - -has_vm_systemd() { - is_command systemd-detect-virt && systemd-detect-virt --help | grep -q -e '^ -v' -} - -check_vm_systemd() ( - _hypervisor=$(systemd-detect-virt -v 2>/dev/null) && - case ${_hypervisor} - in - (''|'none') - return 1 ;; - ('amazon') - echo aws ;; - ('bochs') - echo kvm ;; - ('microsoft') - # assumption - echo hyperv ;; - ('oracle') - echo virtualbox ;; - ('vm-other') - return 0 ;; - (*) - echo "${_hypervisor}" ;; - esac -) - -has_vm_cpuinfo() { has_cpuinfo; } - -check_vm_cpuinfo() { - if grep -q -F 'User Mode Linux' /proc/cpuinfo \ - || grep -q -F 'UML' /proc/cpuinfo - then - # User Mode Linux - echo uml - elif grep -q -e '^vendor_id.*: PowerVM Lx86' /proc/cpuinfo - then - # IBM PowerVM Lx86 (Linux/x86 emulator) - echo powervm_lx86 - elif grep -q -e '^vendor_id.*: IBM/S390' /proc/cpuinfo - then - # IBM SystemZ (S/390) - if test -f /proc/sysinfo - then - if grep -q -e '^VM[0-9]* Control Program: KVM/Linux' /proc/sysinfo - then - echo kvm-s390 - return 0 - elif grep -q -e '^VM[0-9]* Control Program: z/VM' /proc/sysinfo - then - echo zvm - return 0 - elif grep -q -e '^LPAR ' /proc/sysinfo - then - echo zvm-lpar - return 0 - fi - fi - return 1 - else - if grep -q -e '^model name.*:' /proc/cpuinfo - then - sed -n -e 's/^model name[^:]*: *//p' /proc/cpuinfo \ - | while read -r _cpu_model - do - guess_hypervisor_from_cpu_model "${_cpu_model}" - done \ - | sort \ - | uniq -c \ - | awk ' - { if ($1 > most_c) { most_c = $1; most_s = $2 } } - END { - if (most_s) print most_s - exit !most_s - }' \ - && return 0 - fi - return 1 - fi -} - -check_vm_arch_specific() { - case ${arch} - in - (ppc64|ppc64le) - # Check PPC64 LPAR, KVM - - # example /proc/cpuinfo line indicating 'not baremetal' - # platform : pSeries - # - # example /proc/ppc64/lparcfg systemtype line - # system_type=IBM pSeries (emulated by qemu) - - if has_cpuinfo && grep -q -e 'platform.**pSeries' /proc/cpuinfo - then - if test -e /proc/ppc64/lparcfg - then - # Assume LPAR, now detect shared or dedicated - if grep -q -x -F 'shared_processor_mode=1' /proc/ppc64/lparcfg - then - echo powervm-shared - return 0 - else - echo powervm-dedicated - return 0 - fi - fi - fi - ;; - (sparc*) - # Check for SPARC LDoms - - if test -e /dev/mdesc - then - if test -d /sys/class/vlds/ctrl -a -d /sys/class/vlds/sp - then - # control LDom - return 1 - else - # guest LDom - echo ldom-sparc - fi - - # MDPROP=/usr/lib/ldoms/mdprop.py - # if test -x "${MDPROP}" - # then - # if test -n "$("${MDPROP}" -v iodevice device-type=pciex)" - # then - # echo ldoms-root - # echo ldoms-io - # elif test -n "$("${MDPROP}" -v iov-device vf-id=0)" - # then - # echo ldoms-io - # fi - # fi - return 0 - fi - ;; - (i?86|x86*|amd64|i86pc) - # VMM CPUID flag denotes that this system is running under a VMM - if is_oneof "${uname_s}" Darwin - then - get_sysctl machdep.cpu.features | tr ' ' '\n' | grep -qixF VMM \ - && return 0 - fi - if has_cpuinfo \ - && grep -q -i -e '^flags.*:.*\(hypervisor\|vmm\)\( \|$\)' /proc/cpuinfo - then - return 0 - fi - ;; - (ia64) - if test -d /sys/bus/xen -a ! -d /sys/bus/xen-backend - then - # PV-on-HVM drivers installed in a Xen guest - echo xen-hvm - return 0 - fi - ;; - esac - return 1 -} - -has_vm_dmi() { - # Check for various products in SMBIOS/DMI. - # Note that DMI doesn't exist on all architectures (only x86 and some ARM). - # On other architectures the $dmi variable will be empty. - - if test -d /sys/class/dmi/id/ - then - dmi_sysfs=/sys/class/dmi/id - elif test -d /sys/devices/virtual/dmi/id/ - then - dmi_sysfs=/sys/devices/virtual/dmi/id - fi - - # shellcheck disable=SC2015 - { - is_command dmidecode \ - && ( - # dmidecode needs to exit 0 and not print the No SMBIOS/DMI line - dmi_out=$(dmidecode 2>&1) \ - && ! printf '%s\n' "${dmi_out}" \ - | grep -qF 'No SMBIOS nor DMI entry point found, sorry.' - ) \ - || test -d "${dmi_sysfs}" - } -} - -check_vm_dmi() { - case $(get_dmi_field system-product-name) - in - (*.metal) - if test "$(get_dmi_field system-manufacturer)" = 'Amazon EC2' - then - # AWS EC2 bare metal -> no virtualisation - return 1 - fi - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('Google Compute Engine') - echo gce - return 0 - ;; - ('RHEV Hypervisor') - # Red Hat Enterprise Virtualization - echo rhev - return 0 - ;; - ('KVM'|'Bochs'|'KVM Virtual Machine') - echo kvm - return 0 - ;; - ('Parallels Virtual Platform') - echo parallels - return 0 - ;; - ('VirtualBox') - echo virtualbox - return 0 - ;; - ('VMware Virtual Platform') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field system-manufacturer) - in - ('Alibaba'*) - case $(get_dmi_field system-product-name) - in - ('Alibaba Cloud ECS') - echo alibaba-ecs - ;; - (*) - echo alibaba - ;; - esac - return 0 - ;; - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('innotek GmbH'|'Oracle Corporation') - echo virtualbox - return 0 - ;; - ('Joyent') - if test "$(get_dmi_field system-product-name)" = 'SmartDC HVM' - then - # SmartOS KVM - echo kvm-smartdc_hvm - return 0 - fi - ;; - ('Microsoft Corporation'*) - if test "$(get_dmi_field system-product-name)" = 'Virtual Machine' - then - if test -e /proc/irq/7/hyperv \ - || expr "$(get_dmi_field bios-version)" : 'VRTUAL.*' >/dev/null - then - echo hyperv - return 0 - fi - - case $(get_dmi_field system-version) - in - (VPC[0-9]*|VS2005*|*[Vv]irtual*[Pp][Cc]*) - echo virtualpc - return 0 - ;; - (*) - echo hyperv - return 0 - ;; - esac - fi - ;; - ('Nutanix') - # Nutanix AHV. Similar to KVM. - if test "$(get_dmi_field system-product-name)" = 'AHV' - then - echo nutanix_ahv - return 0 - fi - ;; - ('oVirt') - echo ovirt - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('QEMU') - echo qemu - return 0 - ;; - ('VMware, Inc.') - echo vmware - return 0 - ;; - esac - - case $(get_dmi_field bios-vendor) - in - ('Amazon EC2') - # AWS on bare-metal or KVM - echo aws-ec2 - return 0 - ;; - ('BHYVE') - echo bhyve - return 0 - ;; - ('innotek GmbH') - echo virtualbox - return 0 - ;; - ('Parallels Software International Inc.') - echo parallels - return 0 - ;; - ('Xen') - if get_dmi_field bios-version | grep -q -e '\([0-9]\{1,\}\.\)\{2\}amazon' - then - # AWS on Xen - echo aws-xen - return 0 - fi - ;; - esac - - return 1 -} - -check_vm_hyp_specific() { - if is_command vmware-checkvm && vmware-checkvm >/dev/null - then - # vmware-checkvm is provided by VMware's open-vm-tools - echo vmware - return 0 - elif test -d /proc/xen - then - test -r /proc/xen/capabilities && - if grep -q -F 'control_d' /proc/xen/capabilities 2>/dev/null - then - # Xen dom0 - return 1 - else - # Xen domU - echo xen - return 0 - fi - fi - return 1 -} - -has_vm_dt() { - # OpenFirmware/Das U-Boot device-tree - test -d /proc/device-tree -} - -check_vm_dt() { - case ${arch} - in - (arm|aarch64) - if test -r /proc/device-tree/hypervisor/compatible - then - if grep -q -F 'xen' /proc/device-tree/hypervisor/compatible - then - echo xen - return 0 - elif grep -q -F 'vmware' /proc/device-tree/hypervisor/compatible - then - # e.g. VMware ESXi on ARM - echo vmware - return 0 - fi - fi - if glob_exists /proc/device-tree/fw-cfg@*/compatible - then - # qemu,fw-cfg-mmio - sed -e 's/,.*$//' /proc/device-tree/fw-cfg@*/compatible | head -n1 - return 0 - fi - if grep -q -F 'dummy-virt' /proc/device-tree/compatible - then - echo lkvm - return 0 - fi - ;; - (ppc64*) - if test -d /proc/device-tree/hypervisor \ - && grep -qF 'linux,kvm' /proc/device-tree/hypervisor/compatible - then - # We are running as a spapr KVM guest on ppc64 - echo kvm-spapr - return 0 - fi - if test -r /proc/device-tree/ibm,partition-name \ - && test -r /proc/device-tree/hmc-managed\? \ - && test -r /proc/device-tree/chosen/qemu,graphic-width - then - echo powervm - fi - ;; - esac - return 1 -} - -has_vm_sys_hypervisor() { - test -d /sys/hypervisor/ -} - -check_vm_sys_hypervisor() { - test -r /sys/hypervisor/type && - case $(head -n1 /sys/hypervisor/type) - in - (xen) - # Ordinary kernel with pv_ops. There does not seem to be - # enough information at present to tell whether this is dom0 - # or domU. - echo xen - return 0 - ;; - esac - return 1 -} - -check_vm_os_specific() { - _hyp_generic=false - - case ${uname_s} - in - (Darwin) - if hv_vmm_present=$(get_sysctl kern.hv_vmm_present) \ - && test "${hv_vmm_present}" -ne 0 - then - _hyp_generic=true - fi - ;; - (FreeBSD) - # FreeBSD does not have /proc/cpuinfo even when procfs is used. - # Instead there is a sysctl kern.vm_guest. - # Which is 'none' if physical, else the virtualisation. - vm_guest=$(get_sysctl kern.vm_guest | tolower) && - case ${vm_guest} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${vm_guest} - in - (hv) echo hyperv ;; - (vbox) echo virtualbox ;; - (*) echo "${vm_guest}" ;; - esac - return 0 - ;; - esac - ;; - (NetBSD) - machdep_hv=$(get_sysctl machdep.hypervisor | tolower) && - case ${machdep_hv} - in - (none) ;; - (generic) _hyp_generic=true ;; - (*) - # kernel could detect hypervisor - case ${machdep_hv} - in - (hyper-v) echo hyperv ;; - (xenhvm*) echo xen-hvm ;; - (xenpv*) echo xen-pv ;; - (xen*) echo xen ;; - (*) echo "${machdep_hv}" ;; - esac - return 0 - ;; - esac - ;; - (OpenBSD) - if is_command hostctl && glob_exists /dev/pvbus[0-9]* - then - for _pvbus in /dev/pvbus[0-9]* - do - _h_out=$(hostctl -f "${_pvbus}" -t 2>/dev/null) || continue - case $(expr "${_h_out}" : '[^:]*: *\(.*\)$') - in - (KVM) echo kvm ;; - (Hyper-V) echo hyperv ;; - (VMware) echo vmware ;; - (Xen) echo xen ;; - (bhyve) echo bhyve ;; - (OpenBSD) echo openbsd_vmm ;; - esac - return 0 - done - fi - ;; - (SunOS) - diag_conf=$(prtdiag | sed -n -e 's/.*Configuration: *//p' -e '/^$/q') - # NOTE: Don't use -e or -F in Solaris grep - if printf '%s\n' "${diag_conf}" | grep -q -i QEMU - then - echo qemu - return 0 - elif printf '%s\n' "${diag_conf}" | grep -q -i VMware - then - echo vmware - return 0 - fi - ;; - (Linux) - if is_command dmesg - then - while read -r line - do - case ${line} - in - ('Booting paravirtualized kernel on ') - case $(expr "${line}" : '.* kernel on \(.*\)') - in - ('Xen') - echo xen-pv; return 0 ;; - ('bare hardware') - return 1 ;; - esac - ;; - ('Hypervisor detected') - case $(expr "${line}" : '.*: *\(.*\)') - in - ('ACRN') - echo acrn ;; - ('Jailhouse') - echo jailhouse ;; - ('KVM') - echo kvm ;; - ('Microsoft Hyper-V') - echo hyperv ;; - ('VMware') - echo vmware ;; - ('Xen HVM') - echo xen-hvm ;; - ('Xen PV') - echo xen-pv ;; - esac - return 0 - ;; - (lpar:*' under hypervisor') - return 0 ;; - esac - done <<-EOF - $(dmesg 2>/dev/null | awk ' - /Booting paravirtualized kernel on / - /Hypervisor detected: / - /lpar: .* under hypervisor/ - ') - EOF - fi - esac - - # Try to guess hypervisor based on CPU model (sysctl hw.model if available) - if cpu_model=$(get_sysctl hw.model) - then - guess_hypervisor_from_cpu_model "${cpu_model}" && return 0 - fi - - if ${_hyp_generic} - then - # cannot say which hypervisor, but one was detected - return 0 - else - return 1 - fi -} - -run_stage() { - if type "has_$1_$2" >/dev/null 2>&1 - then - "has_$1_$2" - else - true - fi \ - && "check_$1_$2" -} - - -# Execute chroot stages - -for stage in \ - procfs debian_ischroot systemd -do - chrootpnt=$(run_stage chroot ${stage}) || continue - is_chrooted=true - detected_layer 'chroot' - if test -n "${chrootpnt}" - then - echo chroot at "${chrootpnt}" - break - fi -done -if ${is_chrooted:-false} && test -z "${chrootpnt}" -then - # could determine chroot, but not its mount point - echo chroot -fi - - -# Execute container stages - -for stage in \ - systemd pid_1 cgroup files os_specific -do - ctengine=$(run_stage ct ${stage}) || continue - detected_layer 'container' - is_contained=true - if test -n "${ctengine}" - then - echo container on "${ctengine}" - break - fi -done -if ${is_contained:-false} && test -z "${ctengine}" -then - # none of the stages could determine the specific container engine, but - # we are running in some container. - echo container -fi - - -# Execute virtual machine / hypervisor stages - -for stage in \ - systemd os_specific hyp_specific sys_hypervisor dt dmi cpuinfo arch_specific -do - hypervisor=$(run_stage vm ${stage}) || continue - detected_layer 'virtual machine' - is_virtual=true - if test -n "${hypervisor}" - then - echo virtual by "${hypervisor}" - break - fi -done -if ${is_virtual:-false} && test -z "${hypervisor}" -then - # none of the stages could determine the specific hypervisor, but - # we are virtual. - echo virtual -fi - - -# Fallback - -detected_layer physical +case "$os" in + "freebsd") + # FreeBSD does not have /proc/cpuinfo even when procfs is used. + # Instead there is a sysctl kern.vm_guest. + # Which is 'none' if physical, else the virtualisation. + vm_guest="$(sysctl -n kern.vm_guest 2>/dev/null || true)" + if [ -n "${vm_guest}" ]; then + if [ "${vm_guest}" = "none" ]; then + echo "physical" + exit + fi + echo "virtual_by_${vm_guest}" + exit + fi + ;; + + "openbsd") + # OpenBSD can also use the sysctl's: hw.vendor or hw.product. + # Note we can be reasonably sure about a machine being virtualised + # as long as we can identify the virtualisation technology. + # But not so much about it being physical... + # Patches are welcome / reach out if you have better ideas. + for sysctl in hw.vendor hw.product; do + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(sysctl -n "${sysctl}")" + done + ;; + + *) + # Defaulting to linux for compatibility with previous cdist behaviour + + if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then + echo openvz + exit + fi + + if [ -e "/proc/1/environ" ] && + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then + echo lxc + exit + fi + + if [ -r /proc/cpuinfo ]; then + # this should only exist on virtual guest machines, + # tested on vmware, xen, kvm, bhyve + if grep -q "hypervisor" /proc/cpuinfo; then + # this file is aviable in xen guest systems + if [ -r /sys/hypervisor/type ]; then + if grep -q -i "xen" /sys/hypervisor/type; then + echo virtual_by_xen + exit + fi + else + for vendor_file in /sys/class/dmi/id/product_name \ + /sys/class/dmi/id/sys_vendor \ + /sys/class/dmi/id/chasis_vendor; do + if [ -r ${vendor_file} ]; then + # This exits if we can make a reasonable judgement + vendor_string_to_machine_type "$(cat "${vendor_file}")" + fi + done + fi + echo "virtual_by_unknown" + exit + else + echo "physical" + exit + fi + fi + ;; +esac + +echo "unknown" diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index b9232ee4..46d87f3e 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -116,13 +116,6 @@ if [ -f /etc/slackware-version ]; then exit 0 fi -# Appliances - -if grep -q '^Check Point Gaia' /etc/cp-release 2>/dev/null; then - echo checkpoint - exit 0 -fi - uname_s="$(uname -s)" # Assume there is no tr on the client -> do lower case ourselves diff --git a/cdist/conf/explorer/os_release b/cdist/conf/explorer/os_release index ec85046f..6489446b 100644 --- a/cdist/conf/explorer/os_release +++ b/cdist/conf/explorer/os_release @@ -34,9 +34,5 @@ elif test -f /var/run/os-release then # FreeBSD (created by os-release service) cat /var/run/os-release -elif test -f /etc/cp-release -then - # Checkpoint firewall or management (actually linux based) - cat /etc/cp-release fi diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index fc59fd14..bbc9e4f0 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -41,9 +41,6 @@ in # empty, but well... cat /etc/arch-release ;; - checkpoint) - awk '{version=$NF; printf("%s\n", substr(version, 2))}' /etc/cp-release - ;; debian) debian_version=$(cat /etc/debian_version) case $debian_version @@ -82,7 +79,6 @@ in # ceres versions don't have a number, so we decode by codename: case ${devuan_version} in - (daedalus/ceres) echo 4.99 ;; (chimaera/ceres) echo 3.99 ;; (beowulf/ceres) echo 2.99 ;; (ascii/ceres) echo 1.99 ;; diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state index b464179a..b7fe08fa 100755 --- a/cdist/conf/type/__apt_mark/explorer/state +++ b/cdist/conf/type/__apt_mark/explorer/state @@ -24,4 +24,4 @@ else name="$__object_id" fi -apt-mark showhold | grep -q "^${name}$" && echo hold || echo unhold +apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold diff --git a/cdist/conf/type/__apt_pin/man.rst b/cdist/conf/type/__apt_pin/man.rst index e6ec8b51..4229c0cd 100644 --- a/cdist/conf/type/__apt_pin/man.rst +++ b/cdist/conf/type/__apt_pin/man.rst @@ -23,7 +23,7 @@ package Package name, glob or regular expression to match (multiple) packages. If not specified `__object_id` is used. priority - The priority value to assign to matching packages. Defaults to 500. (To match the default target distro's priority) + The priority value to assign to matching packages. Deafults to 500. (To match the default target distro's priority) state Will be passed to underlying `__file` type; see there for valid values and defaults. diff --git a/cdist/conf/type/__apt_pin/manifest b/cdist/conf/type/__apt_pin/manifest index 983b2b42..e72a8fdd 100755 --- a/cdist/conf/type/__apt_pin/manifest +++ b/cdist/conf/type/__apt_pin/manifest @@ -57,11 +57,6 @@ __file "/etc/apt/preferences.d/$name" \ --owner root --group root --mode 0644 \ --state "$state" \ --source - << EOF -# Created by cdist ${__type##*/} -# Do not change. Changes will be overwritten. -# - -# $name Package: $package Pin: $pin Pin-Priority: $priority diff --git a/cdist/conf/type/__apt_pin/parameter/default/priority b/cdist/conf/type/__apt_pin/parameter/default/priority deleted file mode 100644 index 1b79f38e..00000000 --- a/cdist/conf/type/__apt_pin/parameter/default/priority +++ /dev/null @@ -1 +0,0 @@ -500 diff --git a/cdist/conf/type/__apt_pin/parameter/optional b/cdist/conf/type/__apt_pin/parameter/optional index 847e703d..52f01fd2 100644 --- a/cdist/conf/type/__apt_pin/parameter/optional +++ b/cdist/conf/type/__apt_pin/parameter/optional @@ -1,3 +1,2 @@ state package -priority diff --git a/cdist/conf/type/__apt_pin/parameter/required b/cdist/conf/type/__apt_pin/parameter/required index c8572d92..4b4e9741 100644 --- a/cdist/conf/type/__apt_pin/parameter/required +++ b/cdist/conf/type/__apt_pin/parameter/required @@ -1 +1,2 @@ distribution +priority diff --git a/cdist/conf/type/__apt_ppa/files/remove-apt-repository b/cdist/conf/type/__apt_ppa/files/remove-apt-repository new file mode 100755 index 00000000..3eb7d491 --- /dev/null +++ b/cdist/conf/type/__apt_ppa/files/remove-apt-repository @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# +# Remove the given apt repository. +# +# Exit with: +# 0: if it worked +# 1: if not +# 2: on other error + +import os +import sys +from aptsources import distro, sourceslist +from softwareproperties import ppa +from softwareproperties.SoftwareProperties import SoftwareProperties + + +def remove_if_empty(file_name): + with open(file_name, 'r') as f: + if f.read().strip(): + return + os.unlink(file_name) + +def remove_repository(repository): + #print 'repository:', repository + codename = distro.get_distro().codename + #print 'codename:', codename + (line, file) = ppa.expand_ppa_line(repository.strip(), codename) + #print 'line:', line + #print 'file:', file + deb_source_entry = sourceslist.SourceEntry(line, file) + src_source_entry = sourceslist.SourceEntry('deb-src{}'.format(line[3:]), file) + + try: + sp = SoftwareProperties() + sp.remove_source(deb_source_entry) + try: + # If there's a deb-src entry, remove that too + sp.remove_source(src_source_entry) + except: + pass + remove_if_empty(file) + return True + except ValueError: + print >> sys.stderr, "Error: '%s' doesn't exists in a sourcelist file" % line + return False + +if __name__ == '__main__': + if (len(sys.argv) != 2): + print >> sys.stderr, 'Error: need a repository as argument' + sys.exit(2) + repository = sys.argv[1] + if remove_repository(repository): + sys.exit(0) + else: + sys.exit(1) diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index e41341b8..84ebebfe 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -29,9 +29,9 @@ fi case "$state_should" in present) - echo "add-apt-repository -y '$name'" + echo "add-apt-repository '$name'" ;; absent) - echo "add-apt-repository -r -y '$name'" + echo "remove-apt-repository '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index 57e85442..c6f4e876 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -20,4 +20,9 @@ __package software-properties-common +require="__package/software-properties-common" \ + __file /usr/local/bin/remove-apt-repository \ + --source "$__type/files/remove-apt-repository" \ + --mode 0755 + require="$__object_name" __apt_update_index diff --git a/cdist/conf/type/__apt_source/files/source.list.template b/cdist/conf/type/__apt_source/files/source.list.template index a28bb45f..d4420e96 100755 --- a/cdist/conf/type/__apt_source/files/source.list.template +++ b/cdist/conf/type/__apt_source/files/source.list.template @@ -2,14 +2,13 @@ set -u entry="$uri $distribution $component" - cat << DONE # Created by cdist ${__type##*/} # Do not change. Changes will be overwritten. # # $name -deb ${options} $entry +deb ${forcedarch} $entry DONE if [ -f "$__object/parameter/include-src" ]; then echo "deb-src $entry" diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote index 973b0f6c..1e8592c6 100755 --- a/cdist/conf/type/__apt_source/gencode-remote +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -22,21 +22,7 @@ name="$__object_id" destination="/etc/apt/sources.list.d/${name}.list" -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - -# run 'apt-get update' only if something changed with our sources.list file -# it will be run a second time on error as a redundancy messure to success if grep -q "^__file${destination}" "$__messages_in"; then - printf 'apt-get %s update || apt-get %s update\n' "$apt_opts" "$apt_opts" + printf 'apt-get update || apt-get update\n' fi diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index d317a135..d1acb388 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -23,9 +23,6 @@ OPTIONAL PARAMETERS arch set this if you need to force and specific arch (ubuntu specific) -signed-by - provide a GPG key fingerprint or keyring path for signature checks - state 'present' or 'absent', defaults to 'present' @@ -59,11 +56,6 @@ EXAMPLES --uri http://archive.canonical.com/ \ --component partner --state present - __apt_source goaccess \ - --uri http://deb.goaccess.io/ \ - --component main \ - --signed-by C03B48887D5E56B046715D3297BD1A0133449C3D - AUTHORS ------- diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index fd1ec47f..35f15909 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -21,7 +21,6 @@ name="$__object_id" state="$(cat "$__object/parameter/state")" uri="$(cat "$__object/parameter/uri")" -options="" if [ -f "$__object/parameter/distribution" ]; then distribution="$(cat "$__object/parameter/distribution")" @@ -32,15 +31,9 @@ fi component="$(cat "$__object/parameter/component")" if [ -f "$__object/parameter/arch" ]; then - options="arch=$(cat "$__object/parameter/arch")" -fi - -if [ -f "$__object/parameter/signed-by" ]; then - options="$options signed-by=$(cat "$__object/parameter/signed-by")" -fi - -if [ "$options" ]; then - options="[$options]" + forcedarch="[arch=$(cat "$__object/parameter/arch")]" +else + forcedarch="" fi # export variables for use in template @@ -48,7 +41,7 @@ export name export uri export distribution export component -export options +export forcedarch # generate file from template mkdir "$__object/files" diff --git a/cdist/conf/type/__apt_source/parameter/optional b/cdist/conf/type/__apt_source/parameter/optional index 0b5470a1..87537335 100644 --- a/cdist/conf/type/__apt_source/parameter/optional +++ b/cdist/conf/type/__apt_source/parameter/optional @@ -1,5 +1,4 @@ state distribution component -arch -signed-by +arch \ No newline at end of file diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 2d7f9030..70b59710 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -18,23 +18,9 @@ # along with cdist. If not, see . # - -# There are special arguments to apt(8) to prevent aborts if apt woudn't been -# updated after the 19th April 2021 till the bullseye release. The additional -# arguments acknoledge the happend suite change (the apt(8) update does the -# same by itself). -# -# Using '-o $config' instead of the --allow-releaseinfo-change-* parameter -# allows backward compatablility to pre-buster Debian versions. -# -# See more: ticket #861 -# https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 -apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # run 'apt-get update' if anything in /etc/apt is newer then /var/lib/apt/lists -# it will be run a second time on error as a redundancy messure to success cat << DONE if find /etc/apt -mindepth 1 -cnewer /var/lib/apt/lists | grep . > /dev/null; then - apt-get $apt_opts update || apt-get $apt_opts update + apt-get update || apt-get update fi DONE diff --git a/cdist/conf/type/__cdist_preos_trigger/gencode-remote b/cdist/conf/type/__cdist_preos_trigger/gencode-remote new file mode 100644 index 00000000..d5e9fe5c --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/gencode-remote @@ -0,0 +1,12 @@ +#!/bin/sh + +os=$(cat "$__global/explorer/os") + +case "$os" in + devuan) + echo "update-rc.d cdist-preos-trigger defaults > /dev/null" + ;; + *) + ;; +esac + diff --git a/cdist/conf/type/__cdist_preos_trigger/man.rst b/cdist/conf/type/__cdist_preos_trigger/man.rst new file mode 100644 index 00000000..abbd553b --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/man.rst @@ -0,0 +1,45 @@ +cdist-type__cdist_preos_trigger(7) +================================== + +NAME +---- +cdist-type__cdist_preos_trigger - configure cdist preos trigger + + +DESCRIPTION +----------- +Create cdist PreOS trigger by creating systemd unit file that will be started +at boot and will execute trigger command - connect to specified host and port. + + +REQUIRED PARAMETERS +------------------- +trigger-command + Command that will be executed as a PreOS cdist trigger. + + +OPTIONAL PARAMETERS +------------------- +None + + +EXAMPLES +-------- + +.. code-block:: sh + + # Configure default curl trigger for host cdist.ungleich.ch at port 80. + __cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80' + + +AUTHORS +------- +Darko Poljak + + +COPYING +------- +Copyright \(C) 2016 Darko Poljak. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__cdist_preos_trigger/manifest b/cdist/conf/type/__cdist_preos_trigger/manifest new file mode 100644 index 00000000..750a0548 --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/manifest @@ -0,0 +1,67 @@ +#!/bin/sh + +os="$(cat "$__global/explorer/os")" +trigger_command=$(cat "$__object/parameter/trigger-command") + +case "$os" in + devuan) + __file /etc/init.d/cdist-preos-trigger --owner root \ + --group root \ + --mode 755 \ + --source - << EOF +#!/bin/sh +# /etc/init.d/cdist-preos-trigger + +### BEGIN INIT INFO +# Provides: cdist-preos-trigger +# Required-Start: \$all +# Required-Stop: +# Default-Start: 2 3 4 5 S +# Default-Stop: 0 1 6 +# Short-Description: Execute cdist preos trigger command +# Description: Execute cdist preos trigger commnad. +### END INIT INFO + +case "\$1" in + start) + echo "Starting cdist-preos-trigger command" + ${trigger_command} & + ;; + stop) + # no-op + ;; + *) + echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}" + exit 1 + ;; +esac + +exit 0 +EOF + ;; + *) + __file /etc/systemd/system/cdist-preos-trigger.service --owner root \ + --group root \ + --mode 644 \ + --source - << EOF +[Unit] +Description=preos trigger +Wants=network-online.target +After=network.target network-online.target + +[Service] +Type=simple +Restart=no +# Broken systemd +ExecStartPre=/bin/sleep 5 +ExecStart=${trigger_command} + +[Install] +WantedBy=multi-user.target +EOF + + require="__file/etc/systemd/system/cdist-preos-trigger.service" \ + __start_on_boot cdist-preos-trigger + ;; +esac + diff --git a/cdist/conf/type/__cdist_preos_trigger/parameter/required b/cdist/conf/type/__cdist_preos_trigger/parameter/required new file mode 100644 index 00000000..3407a482 --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/parameter/required @@ -0,0 +1 @@ +trigger-command diff --git a/cdist/conf/type/__debconf_set_selections/parameter/optional b/cdist/conf/type/__debconf_set_selections/parameter/deprecated similarity index 100% rename from cdist/conf/type/__debconf_set_selections/parameter/optional rename to cdist/conf/type/__debconf_set_selections/parameter/deprecated diff --git a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file b/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file deleted file mode 100644 index 09db545a..00000000 --- a/cdist/conf/type/__debconf_set_selections/parameter/deprecated/file +++ /dev/null @@ -1 +0,0 @@ -'file' has been deprecated in favour of 'line' in order to provide idempotency. diff --git a/cdist/conf/type/__dot_file/man.rst b/cdist/conf/type/__dot_file/man.rst index c8f36712..ba7621a1 100644 --- a/cdist/conf/type/__dot_file/man.rst +++ b/cdist/conf/type/__dot_file/man.rst @@ -37,12 +37,6 @@ state source forwarded to :strong:`__file` type -file - forwarded to :strong:`__file` type - This can be used if multiple users need to have a dotfile updated, - which will result in duplicate object id errors. When using the - file parameter the object id can be some unique value. - MESSAGES -------- @@ -67,15 +61,6 @@ EXAMPLES # Install default xmonad config for user 'eve'. Parent directory is created automatically. __dot_file .xmonad/xmonad.hs --user eve --state exists --source "$__files/xmonad.hs" - # install .vimrc for root and some users - for user in root userx usery userz; do - __dot_file "${user}_dot_vimrc" \ - --user $user \ - --file .vimrc \ - --state exists \ - --source "$__files/$user/.vimrc" - done - SEE ALSO -------- diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index a38ed943..02dadf05 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -20,19 +20,13 @@ user="$(cat "${__object}/parameter/user")" home="$(cat "${__object}/explorer/home")" primary_group="$(cat "${__object}/explorer/primary_group")" dirmode="$(cat "${__object}/parameter/dirmode")" -if [ -f "${__object}/parameter/file" ]; then - file="$(cat "${__object}/parameter/file")" -else - file="${__object_id}" -fi - # Create parent directory. Type __directory has flag 'parents', but it # will leave us with root-owned directory in user home, which is not # acceptable. So we create parent directories one-by-one. XXX: maybe # it should be fixed in '__directory'? set -- -subpath=${file} +subpath=${__object_id} while subpath="$(dirname "${subpath}")" ; do [ "${subpath}" = . ] && break set -- "${subpath}" "$@" @@ -70,4 +64,4 @@ if [ "${source}" = "-" ] ; then fi unset source -__file "${home}/${file}" --owner "$user" --group "$primary_group" "$@" +__file "${home}/${__object_id}" --owner "$user" --group "$primary_group" "$@" diff --git a/cdist/conf/type/__file/__init__.py b/cdist/conf/type/__file/__init__.py new file mode 100644 index 00000000..bcb525fc --- /dev/null +++ b/cdist/conf/type/__file/__init__.py @@ -0,0 +1,124 @@ +import os +import re +from cdist.core.pytypes import * +import argparse + + +class FileType(PythonType): + def get_attribute(self, stat_file, attribute, value_should): + if os.path.exists(stat_file): + if re.match('[0-9]', value_should): + index = 1 + else: + index = 2 + with open(stat_file, 'r') as f: + for line in f: + if re.match(attribute + ":", line): + fields = line.split() + return fields[index] + return None + + def set_attribute(self, attribute, value_should, destination): + cmd = { + 'group': 'chgrp', + 'owner': 'chown', + 'mode': 'chmod', + } + self.send_message("{} '{}'".format(cmd[attribute], value_should)) + return "{} '{}' '{}'".format(cmd[attribute], value_should, destination) + + def type_manifest(self): + yield from () + + def type_gencode(self): + typeis = self.get_explorer('type') + state_should = self.get_parameter('state') + + if state_should == 'exists' and typeis == 'file': + return + + source = self.get_parameter('source') + if source == '-': + source = self.stdin_path + destination = '/' + self.object_id + if state_should == 'pre-exists': + if source is not None: + self.die('--source cannot be used with --state pre-exists') + if typeis == 'file': + return None + else: + self.die('File {} does not exist'.format(destination)) + + create_file = False + upload_file = False + set_attributes = False + fire_onchange = False + code = [] + if state_should == 'present' or state_should == 'exists': + if source is None: + remote_stat = self.get_explorer('stat') + if not remote_stat: + create_file = True + else: + if os.path.exists(source): + if typeis == 'file': + local_cksum = self.run_local(['cksum', source, ]) + local_cksum = local_cksum.split()[0] + remote_cksum = self.get_explorer('cksum') + remote_cksum = remote_cksum.split()[0] + upload_file = local_cksum != remote_cksum + else: + upload_file = True + else: + self.die('Source {} does not exist'.format(source)) + if create_file or upload_file: + set_attributes = True + fire_onchange = True + tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination) + destination_upload = self.run_remote( + ["mktemp", tempfile_template, ]) + if upload_file: + self.transfer(source, destination_upload) + code.append('rm -rf {}'.format(destination)) + code.append('mv {} {}'.format(destination_upload, destination)) + + if state_should in ('present', 'exists', 'pre-exists', ): + for attribute in ('group', 'owner', 'mode', ): + if attribute in self.parameters: + value_should = self.get_parameter(attribute) + if attribute == 'mode': + value_should = re.sub('^0', '', value_should) + stat_file = self.get_explorer_file('stat') + value_is = self.get_attribute(stat_file, attribute, + value_should) + if set_attributes or value_should != value_is: + fire_onchange = True + code.append(self.set_attribute(attribute, + value_should, + destination)) + elif state_should == 'absent': + if typeis == 'file': + code.append('rm -f {}'.format(destination)) + self.send_message('remove') + fire_onchange = True + else: + self.die('Unknown state {}'.format(state_should)) + + if fire_onchange: + onchange = self.get_parameter('onchange') + if onchange: + code.append(onchange) + + return "\n".join(code) + + def get_args_parser(self): + parser = argparse.ArgumentParser(add_help=False, + argument_default=argparse.SUPPRESS) + parser.add_argument('--state', dest='state', action='store', + required=False, default='present') + for param in ('group', 'mode', 'owner', 'source', 'onchange'): + parser.add_argument('--' + param, dest=param, action='store', + required=False, default=None) + + parser.add_argument("object_id", nargs=1) + return parser diff --git a/cdist/conf/type/__file_old/explorer/cksum b/cdist/conf/type/__file_old/explorer/cksum new file mode 100755 index 00000000..335e4e7a --- /dev/null +++ b/cdist/conf/type/__file_old/explorer/cksum @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Retrieve the md5sum of a file to be created, if it is already existing. +# + +destination="/$__object_id" + +if [ -e "$destination" ]; then + if [ -f "$destination" ]; then + cksum < "$destination" + else + echo "NO REGULAR FILE" + fi +else + echo "NO FILE FOUND, NO CHECKSUM CALCULATED." +fi diff --git a/cdist/conf/type/__file_old/explorer/stat b/cdist/conf/type/__file_old/explorer/stat new file mode 100755 index 00000000..91c8cc84 --- /dev/null +++ b/cdist/conf/type/__file_old/explorer/stat @@ -0,0 +1,116 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +fallback() { + # Fallback: Patch the output together, manually. + + ls_line=$(ls -ldn "$destination") + + uid=$(echo "$ls_line" | awk '{ print $3 }') + gid=$(echo "$ls_line" | awk '{ print $4 }') + + owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd) + group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) + + mode_text=$(echo "$ls_line" | awk '{ print $1 }') + mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }') + + size=$(echo "$ls_line" | awk '{ print $5 }') + links=$(echo "$ls_line" | awk '{ print $2 }') + + printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\nsize: %d\nlinks: %d\n' \ + "$("$__type_explorer/type")" \ + "$uid" "$owner" \ + "$gid" "$group" \ + "$mode" "$mode_text" \ + "$size" \ + "$links" +} + + +# nothing to work with, nothing we could do +[ -e "$destination" ] || exit 0 + + +if ! command -v stat >/dev/null +then + fallback + exit +fi + + +case $("$__explorer/os") +in + freebsd|netbsd|openbsd|macosx) + stat -f "type: %HT +owner: %Du %Su +group: %Dg %Sg +mode: %Lp %Sp +size: %Dz +links: %Dl +" "$destination" | awk '/^type/ { print tolower($0); next } { print }' + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" + echo "size: $( echo "$ls1" | awk '{print $5}' )" + echo "links: $( echo "$ls1" | awk '{print $2}' )" + ;; + *) + # NOTE: Do not use --printf here as it is not supported by BusyBox stat. + # NOTE: BusyBox's stat might not support the "-c" option, in which case + # we fall through to the shell fallback. + stat -c "type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h" "$destination" 2>/dev/null || fallback + ;; +esac diff --git a/cdist/conf/type/__file_old/explorer/type b/cdist/conf/type/__file_old/explorer/type new file mode 100755 index 00000000..e723047c --- /dev/null +++ b/cdist/conf/type/__file_old/explorer/type @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +if [ ! -e "$destination" ]; then + echo none +elif [ -h "$destination" ]; then + echo symlink +elif [ -f "$destination" ]; then + echo file +elif [ -d "$destination" ]; then + echo directory +else + echo unknown +fi diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file_old/gencode-local similarity index 68% rename from cdist/conf/type/__file/gencode-local rename to cdist/conf/type/__file_old/gencode-local index 5a303308..231b6927 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file_old/gencode-local @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -72,7 +72,6 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ "$type" != "file" ]; then # destination is not a regular file, upload source to replace it upload_file=1 - echo upload >> "$__messages_out" else local_cksum="$(cksum < "$source")" remote_cksum="$(cat "$__object/explorer/cksum")" @@ -89,39 +88,27 @@ if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then mkdir "$__object/files" touch "$__object/files/set-attributes" - if [ "$create_file" ]; then - # When creating an empty file we create it locally and then - # upload it so that permissions can be set before moving the file - # into place. - source="$__object/files/empty" - touch "$source" - fi - # upload file to temp location - upload_destination="${destination}.cdist.${__cdist_object_marker}.$$" - # Yes, we are aware that this is a race condition. - # However: - # a) cdist usually writes to directories that are not user writable - # (probably > 99.9%) - # b) if they are user owned, the user / attacker always wins - # (probably < 0.1%) - # c) the only case which we could improve are tmp directories and we - # don't think managing tmp directories with cdist is a typical case - # ("the rest %)" - - # Tell gencode-remote to where we uploaded the file so it can move - # it to its final destination. - echo "$upload_destination" > "$__object/files/upload-destination" - - # IPv6 fix - if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - then - my_target_host="[${__target_host}]" - else - my_target_host="${__target_host}" - fi + tempfile_template="${destination}.cdist.XXXXXXXXXX" cat << DONE -$__remote_copy "$source" "${my_target_host}:${upload_destination}" +destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template")" +DONE + if [ "$upload_file" ]; then + echo upload >> "$__messages_out" + # IPv6 fix + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi + cat << DONE +$__remote_copy "$source" "${my_target_host}:\$destination_upload" +DONE + fi +# move uploaded file into place +cat << DONE +$__remote_exec $__target_host "rm -rf \"$destination\"; mv \"\$destination_upload\" \"$destination\"" DONE fi fi diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file_old/gencode-remote similarity index 82% rename from cdist/conf/type/__file/gencode-remote rename to cdist/conf/type/__file_old/gencode-remote index 1a9ff69c..f7a528fd 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file_old/gencode-remote @@ -1,7 +1,7 @@ #!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2013-2022 Steven Armstrong (steven-cdist armstrong.cc) +# 2013 Steven Armstrong (steven-cdist armstrong.cc) # # This file is part of cdist. # @@ -62,13 +62,6 @@ set_mode() { case "$state_should" in present|exists) - if [ -f "$__object/files/upload-destination" ]; then - final_destination="$destination" - # We change the 'global' $destination variable here so we can - # change attributes of the new/uploaded file before moving it - # to it's final destination. - destination="$(cat "$__object/files/upload-destination")" - fi # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -88,11 +81,6 @@ case "$state_should" in fi fi done - if [ -f "$__object/files/upload-destination" ]; then - # move uploaded file into place - printf 'rm -rf "%s"\n' "$final_destination" - printf 'mv "%s" "%s"\n' "$destination" "$final_destination" - fi if [ -f "$__object/files/set-attributes" ]; then # set-attributes is created if file is created or uploaded in gencode-local fire_onchange=1 diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file_old/man.rst similarity index 100% rename from cdist/conf/type/__file/man.rst rename to cdist/conf/type/__file_old/man.rst diff --git a/cdist/conf/type/__file/parameter/default/state b/cdist/conf/type/__file_old/parameter/default/state similarity index 100% rename from cdist/conf/type/__file/parameter/default/state rename to cdist/conf/type/__file_old/parameter/default/state diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file_old/parameter/optional similarity index 100% rename from cdist/conf/type/__file/parameter/optional rename to cdist/conf/type/__file_old/parameter/optional diff --git a/cdist/conf/type/__file_py b/cdist/conf/type/__file_py new file mode 120000 index 00000000..efa910bd --- /dev/null +++ b/cdist/conf/type/__file_py @@ -0,0 +1 @@ +__file \ No newline at end of file diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index 0d944482..d145c4c3 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -15,7 +15,7 @@ case $os in # Differntation not needed anymore apt_source_distribution=stable ;; - 10*|11*) + 10*) # Differntation not needed anymore apt_source_distribution=stable ;; diff --git a/cdist/conf/type/__haproxy_dualstack/files/http b/cdist/conf/type/__haproxy_dualstack/files/http deleted file mode 100644 index 0508a465..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/http +++ /dev/null @@ -1,8 +0,0 @@ -frontend http - bind BIND@:80 - mode http - option httplog - default_backend http - -backend http - mode http diff --git a/cdist/conf/type/__haproxy_dualstack/files/https b/cdist/conf/type/__haproxy_dualstack/files/https deleted file mode 100644 index 73deac46..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/https +++ /dev/null @@ -1,10 +0,0 @@ -frontend https - bind BIND@:443 - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend https - -backend https - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/imaps b/cdist/conf/type/__haproxy_dualstack/files/imaps deleted file mode 100644 index b1ec3793..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/imaps +++ /dev/null @@ -1,12 +0,0 @@ -frontend imaps - bind BIND@:143 - bind BIND@:993 - - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend imaps - -backend imaps - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/files/smtps b/cdist/conf/type/__haproxy_dualstack/files/smtps deleted file mode 100644 index dce6ed4a..00000000 --- a/cdist/conf/type/__haproxy_dualstack/files/smtps +++ /dev/null @@ -1,12 +0,0 @@ -frontend smtps - bind BIND@:25 - bind BIND@:465 - - mode tcp - option tcplog - tcp-request inspect-delay 5s - tcp-request content accept if { req_ssl_hello_type 1 } - default_backend smtps - -backend smtps - mode tcp diff --git a/cdist/conf/type/__haproxy_dualstack/man.rst b/cdist/conf/type/__haproxy_dualstack/man.rst deleted file mode 100644 index 6c131cbe..00000000 --- a/cdist/conf/type/__haproxy_dualstack/man.rst +++ /dev/null @@ -1,121 +0,0 @@ -cdist-type__haproxy_dualstack(7) -================================ - - -NAME ----- -cdist-type__haproxy_dualstack - Proxy services from a dual-stack server - - -DESCRIPTION ------------ -This (singleton) type installs and configures haproxy to act as a dual-stack -proxy for single-stack services. - -This can be useful to add IPv4 support to IPv6-only services while only using -one IPv4 for many such services. - -By default this type uses the plain TCP proxy mode, which means that there is no -need for TLS termination on this host when SNI is supported. -This also means that proxied services will not receive the client's IP address, -but will see the proxy's IP address instead (that of `$__target_host`). - -This can be solved by using the PROXY protocol, but do take into account that, -e.g. nginx cannot serve both regular HTTP(S) and PROXY protocols on the same -port, so you will need to use other ports for that. - -As a recommendation in this type: use TCP ports 8080 and 591 respectively to -serve HTTP and HTTPS using the PROXY protocol. - -See the EXAMPLES for more details. - - -OPTIONAL PARAMETERS -------------------- -v4proxy - Proxy incoming IPv4 connections to the equivalent IPv6 endpoint. - In its simplest use, it must be a NAME with an `AAAA` DNS entry, which is - the IP address actually providing the proxied services. - The full format of this argument is: - `[proxy:]NAME[[:PROTOCOL_1=PORT_1]...[:PROTOCOL_N=PORT_N]]` - Where starting with `proxy:` determines that the PROXY protocol must be - used and each `:PROTOCOL=PORT` (e.g. `:http=8080` or `:https=591`) is a PORT - override for the given PROTOCOL (see `--protocol`), if not present the - PROTOCOL's default port will be used. - - -v6proxy - Proxy incoming IPv6 connections to the equivalent IPv4 endpoint. - In its simplest use, it must be a NAME with an `A` DNS entry, which is - the IP address actually providing the proxied services. - See `--v4proxy` for more options and details. - -protocol - Can be passed multiple times or as a space-separated list of protocols. - Currently supported protocols are: `http`, `https`, `imaps`, `smtps`. - This defaults to: `http https imaps smtps`. - - -EXAMPLES --------- - -.. code-block:: sh - - # Proxy the IPv6-only services so IPv4-only clients can access them - # This uses HAProxy's TCP mode for http, https, imaps and smtps - __haproxy_dualstack \ - --v4proxy ipv6.chat \ - --v4proxy matrix.ungleich.ch - - # Proxy the IPv6-only HTTP(S) services so IPv4-only clients can access them - # Note this means that the backend IPv6-only server will only see - # the IPv6 address of the haproxy host managed by cdist, which can be - # troublesome if this information is relevant for analytics/security/... - # See the PROXY example below - __haproxy_dualstack \ - --protocol http --protocol https \ - --v4proxy ipv6.chat \ - --v4proxy matrix.ungleich.ch - - # Use the PROXY protocol to proxy the IPv6-only HTTP(S) services enabling - # IPv4-only clients to access them while maintaining the client's IP address - __haproxy_dualstack \ - --protocol http --protocol https \ - --v4proxy proxy:ipv6.chat:http=8080:https=591 \ - --v4proxy proxy:matrix.ungleich.ch:http=8080:https=591 - # Note however that the PROXY protocol is not compatible with regular - # HTTP(S) protocols, so your nginx will have to listen on different ports - # with the PROXY settings. - # Note that you will need to restrict access to the 8080 port to prevent - # Client IP spoofing. - # This can be something like: - # server { - # # listen for regular HTTP connections - # listen [::]:80 default_server; - # listen 80 default_server; - # # listen for PROXY HTTP connections - # listen [::]:8080 proxy_protocol; - # # Accept the Client's IP from the PROXY protocol - # real_ip_header proxy_protocol; - # } - - -SEE ALSO --------- -- https://www.haproxy.com/blog/enhanced-ssl-load-balancing-with-server-name-indication-sni-tls-extension/ -- https://www.haproxy.com/blog/haproxy/proxy-protocol/ -- https://docs.nginx.com/nginx/admin-guide/load-balancer/using-proxy-protocol/ - - -AUTHORS -------- -ungleich -Evilham - - -COPYING -------- -Copyright \(C) 2021 ungleich glarus ag. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__haproxy_dualstack/manifest b/cdist/conf/type/__haproxy_dualstack/manifest deleted file mode 100644 index d110eea6..00000000 --- a/cdist/conf/type/__haproxy_dualstack/manifest +++ /dev/null @@ -1,155 +0,0 @@ -#!/bin/sh -eu - -__package haproxy -require="__package/haproxy" __start_on_boot haproxy - -tmpdir="$__object/files" -mkdir "$tmpdir" -configtmp="$__object/files/haproxy.cfg" - -os=$(cat "$__global/explorer/os") -case $os in - freebsd) - CONFIG_FILE="/usr/local/etc/haproxy.conf" - cat < "$configtmp" -global - maxconn 4000 - user nobody - group nogroup - daemon - -EOF - - ;; - *) - CONFIG_FILE="/etc/haproxy/haproxy.cfg" - cat < "$configtmp" -global - log [::1] local2 - chroot /var/lib/haproxy - pidfile /var/run/haproxy.pid - maxconn 4000 - user haproxy - group haproxy - daemon - - # turn on stats unix socket - stats socket /var/lib/haproxy/stats - -EOF - ;; -esac - -cat <> "$configtmp" -defaults - retries 3 - log global - timeout http-request 10s - timeout queue 1m - timeout connect 10s - timeout client 1m - timeout server 1m - timeout http-keep-alive 10s - timeout check 10s -EOF - -dig_cmd="$(command -v dig || true)" -get_ip() { - # Usage: get_ip (ipv4|ipv6) NAME - # uses "dig" if available, else fallback to "host" - case $1 in - ipv4) - if [ -n "${dig_cmd}" ]; then - ${dig_cmd} +short A "$2" - else - host -t A "$2" | cut -d ' ' -f 4 | grep -v 'found:' - fi - ;; - ipv6) - if [ -n "${dig_cmd}" ]; then - ${dig_cmd} +short AAAA "$2" - else - host -t AAAA "$2" | cut -d ' ' -f 5 | grep -v 'NXDOMAIN' - fi - ;; - esac -} - -PROTOCOLS="$(cat "$__object/parameter/protocol")" - -for proxy in v4proxy v6proxy; do - param=$__object/parameter/$proxy - # no backend? skip generating code - if [ ! -f "$param" ]; then - continue - fi - - # turn backend name into bind parameter: v4backend -> ipv4@ - bind=$(echo $proxy | sed -e 's/^/ip/' -e 's/proxy//') - - case $bind in - ipv4) - backendproto=ipv6 - ;; - ipv6) - backendproto=ipv4 - ;; - esac - - for proto in ${PROTOCOLS}; do - # Add protocol "header" - printf "\n# %s %s \n" "${bind}" "${proto}" >> "$configtmp" - - sed -e "s/BIND/$bind/" \ - -e "s/\(frontend[[:space:]].*\)/\1$bind/" \ - -e "s/\(backend[[:space:]].*\)/\\1$bind/" \ - "$__type/files/$proto" >> "$configtmp" - - while read -r hostdefinition; do - if echo "$hostdefinition" | grep -qE '^proxy:'; then - # Proxy protocol was requested - host="$(echo "$hostdefinition" | sed -E 's/^proxy:([^:]+).*$/\1/')" - send_proxy=" send-proxy" - else - # Just use tcp proxy mode - host="$hostdefinition" - send_proxy="" - fi - if echo "$hostdefinition" | grep -qE ":${proto}="; then - # Use custom port definition if requested - port="$(echo "$hostdefinition" | sed -E "s/^(.*:)?${proto}=([0-9]+).*$/:\2/")" - else - # Else use the default - port="" - fi - servername=$host - - res=$(get_ip "$bind" "$servername") - - if [ -z "$res" ]; then - echo "$servername does not resolve - aborting config" >&2 - exit 1 - fi - - # Treat protocols without TLS+SNI specially - if [ "$proto" = http ]; then - echo " use-server $servername if { hdr(host) -i $host }" >> "$configtmp" - else - echo " use-server $servername if { req_ssl_sni -i $host }" >> "$configtmp" - fi - - # Create the "server" itself. - # Note that port and send_proxy will be empty unless - # they were requested by the type user - echo " server $servername ${backendproto}@${host}${port}${send_proxy}" >> "$configtmp" - - done < "$param" - done -done - -# Create config file -require="__package/haproxy" __file ${CONFIG_FILE} --source "$configtmp" --mode 0644 - -require="__file${CONFIG_FILE}" __check_messages "haproxy_reload" \ - --pattern "^__file${CONFIG_FILE}" \ - --execute "service haproxy reload || service haproxy restart" diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol b/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol deleted file mode 100644 index dc8bb7bf..00000000 --- a/cdist/conf/type/__haproxy_dualstack/parameter/default/protocol +++ /dev/null @@ -1 +0,0 @@ -http https imaps smtps diff --git a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple b/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple deleted file mode 100644 index 8c482bd4..00000000 --- a/cdist/conf/type/__haproxy_dualstack/parameter/optional_multiple +++ /dev/null @@ -1,3 +0,0 @@ -protocol -v4proxy -v6proxy diff --git a/cdist/conf/type/__haproxy_dualstack/singleton b/cdist/conf/type/__haproxy_dualstack/singleton deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest index 39067f3b..6394f629 100644 --- a/cdist/conf/type/__letsencrypt_cert/manifest +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -41,7 +41,7 @@ if [ -z "${certbot_fullpath}" ]; then require="__apt_source/stretch-backports" __package_apt certbot \ --target-release stretch-backports ;; - 10*|11*) + 10*) __package_apt certbot ;; @@ -85,7 +85,7 @@ if [ -z "${certbot_fullpath}" ]; then esac ;; freebsd) - __package py39-certbot + __package py37-certbot certbot_fullpath="/usr/local/bin/certbot" ;; ubuntu) diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index 79c0d9d3..fbfca330 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -81,24 +81,12 @@ aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes -o Dpkg::Options::= case "$state_should" in present) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - # following is bit ugly, but important hack. # due to how cdist config run works, there isn't # currently better way to do it :( cat << EOF if [ ! -f /var/cache/apt/pkgcache.bin ] || [ "\$( stat --format %Y /var/cache/apt/pkgcache.bin )" -lt "\$( date +%s -d '-1 day' )" ] -then echo apt-get $apt_opts update > /dev/null 2>&1 || true +then echo apt-get update > /dev/null 2>&1 || true fi EOF if [ -n "$version" ]; then diff --git a/cdist/conf/type/__package_luarocks/manifest b/cdist/conf/type/__package_luarocks/manifest index 9e4499b2..7d8262ca 100755 --- a/cdist/conf/type/__package_luarocks/manifest +++ b/cdist/conf/type/__package_luarocks/manifest @@ -19,5 +19,5 @@ # along with cdist. If not, see . # -__package luarocks --state present -__package make --state present +__package luarocks --state installed +__package make --state installed diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index a10c16d3..803468b5 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -41,19 +41,7 @@ fi case "$type" in yum) ;; apt) - # There are special arguments to apt(8) to prevent aborts if apt woudn't been - # updated after the 19th April 2021 till the bullseye release. The additional - # arguments acknoledge the happend suite change (the apt(8) update does the - # same by itself). - # - # Using '-o $config' instead of the --allow-releaseinfo-change-* parameter - # allows backward compatablility to pre-buster Debian versions. - # - # See more: ticket #861 - # https://code.ungleich.ch/ungleich-public/cdist/-/issues/861 - apt_opts="-o Acquire::AllowReleaseInfoChange::Suite=true -o Acquire::AllowReleaseInfoChange::Version=true" - - echo "apt-get --quiet $apt_opts update" + echo "apt-get --quiet update" echo "apt-cache updated (age was: $currage)" >> "$__messages_out" ;; pacman) diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index d332e851..38aa001e 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -28,10 +28,6 @@ apt_clean="$__object/parameter/apt-clean" apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" -if [ -f "$__object/parameter/apt-with-new-pkgs" ]; then - apt_with_new_pkgs="--with-new-pkgs" -fi - if [ -f "$type" ]; then type="$(cat "$type")" else @@ -58,7 +54,7 @@ case "$type" in apt) if [ -f "$apt_dist_upgrade" ] then echo "$aptget dist-upgrade" - else echo "$aptget $apt_with_new_pkgs upgrade" + else echo "$aptget upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index 0c116bac..e9e2b8ce 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -33,14 +33,6 @@ BOOLEAN PARAMETERS apt-dist-upgrade Do dist-upgrade instead of upgrade. -apt-with-new-pkg - Allow installing new packages when used in conjunction with - upgrade. This is useful if the update of an installed package - requires new dependencies to be installed. Instead of holding the - package back upgrade will upgrade the package and install the new - dependencies. Note that upgrade with this option will never remove - packages, only allow adding new ones. - apt-clean Clean out the local repository of retrieved package files. diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean index cd22eb90..7a56a34b 100644 --- a/cdist/conf/type/__package_upgrade_all/parameter/boolean +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -1,3 +1,2 @@ apt-clean apt-dist-upgrade -apt-with-new-pkgs diff --git a/cdist/conf/type/__sed/explorer/file b/cdist/conf/type/__sed/explorer/file deleted file mode 100755 index ec3d0fe8..00000000 --- a/cdist/conf/type/__sed/explorer/file +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -if [ ! -e "$file" ] -then - echo "$file does not exist" >&2 - exit 1 -fi - -cat "$file" diff --git a/cdist/conf/type/__sed/gencode-remote b/cdist/conf/type/__sed/gencode-remote deleted file mode 100755 index f99c5a88..00000000 --- a/cdist/conf/type/__sed/gencode-remote +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh -e - -if [ -f "$__object/parameter/file" ] -then - file="$( cat "$__object/parameter/file" )" -else - file="/$__object_id" -fi - -script="$( cat "$__object/parameter/script" )" - -if [ "$script" = '-' ] -then - script="$( cat "$__object/stdin" )" -fi - -# since stdin is not available in explorer, we pull file from target with explorer - -file_from_target="$__object/explorer/file" - -sed_cmd='sed' - -if [ -f "$__object/parameter/regexp-extended" ] -then - sed_cmd="$sed_cmd -E" -fi - -# do sed dry run, diff result and if no change, then there's nothing to do -# also redirect diff's output to stderr for debugging purposes - -if echo "$script" | "$sed_cmd" -f - "$file_from_target" | diff -u "$file_from_target" - >&2 -then - exit 0 -fi - -# we can't use -i, because it's not posix, so we fly with tempfile and cp -# and we use cp because we want to preserve destination file's attributes - -# shellcheck disable=SC2016 -echo 'tmp="$__object/tempfile"' - -echo "$sed_cmd -f - '$file' > \"\$tmp\" << EOF" - -echo "$script" - -echo 'EOF' - -echo "cp \"\$tmp\" '$file'" - -# shellcheck disable=SC2016 -echo 'rm -f "$tmp"' - -echo 'change' >> "$__messages_out" - -if [ -f "$__object/parameter/onchange" ] -then - cat "$__object/parameter/onchange" -fi diff --git a/cdist/conf/type/__sed/man.rst b/cdist/conf/type/__sed/man.rst deleted file mode 100644 index 86789363..00000000 --- a/cdist/conf/type/__sed/man.rst +++ /dev/null @@ -1,57 +0,0 @@ -cdist-type__sed(7) -================== - -NAME ----- -cdist-type__sed - Transform text files with ``sed`` - - -DESCRIPTION ------------ -Transform text files with ``sed``. - - -REQUIRED MULTIPLE PARAMETERS ----------------------------- -script - ``sed`` script. - If ``-`` then the script is read from ``stdin``. - - -OPTIONAL PARAMETERS -------------------- -file - Path to the file. Defaults to ``$__object_id``. - -onchange - Execute this command if ``sed`` changes file. - - -BOOLEAN PARAMETERS ------------------- -regexp-extended - Use extended regular expressions in the script. - Might not be supported with every ``sed`` version. - - -EXAMPLES --------- - -.. code-block:: sh - - __sed /tmp/foobar --script 's/foo/bar/' - - echo 's/foo/bar/' | __sed foobar --file /tmp/foobar --script - - - -AUTHORS -------- -Ander Punnar - - -COPYING -------- -Copyright \(C) 2021 Ander Punnar. You can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the Free -Software Foundation, either version 3 of the License, or (at your option) -any later version. diff --git a/cdist/conf/type/__sed/parameter/boolean b/cdist/conf/type/__sed/parameter/boolean deleted file mode 100644 index 1ad75c5d..00000000 --- a/cdist/conf/type/__sed/parameter/boolean +++ /dev/null @@ -1 +0,0 @@ -regexp-extended diff --git a/cdist/conf/type/__sed/parameter/optional b/cdist/conf/type/__sed/parameter/optional deleted file mode 100644 index fa86f917..00000000 --- a/cdist/conf/type/__sed/parameter/optional +++ /dev/null @@ -1,2 +0,0 @@ -file -onchange diff --git a/cdist/conf/type/__sed/parameter/required_multiple b/cdist/conf/type/__sed/parameter/required_multiple deleted file mode 100644 index 84f7e31d..00000000 --- a/cdist/conf/type/__sed/parameter/required_multiple +++ /dev/null @@ -1 +0,0 @@ -script diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index cbffde94..61c77fb9 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -40,7 +40,6 @@ if [ -f "$file" ]; then grep -v -F -x '$line' '$file' >\$tmpfile fi cat "\$tmpfile" >"$file" -rm -f "\$tmpfile" DONE } diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote index d8612986..b685c990 100755 --- a/cdist/conf/type/__timezone/gencode-remote +++ b/cdist/conf/type/__timezone/gencode-remote @@ -34,12 +34,3 @@ case "$os" in echo "echo \"$timezone_should\" > /etc/timezone" ;; esac - -case "$os" in - openwrt) - cat </dev/null || true +(id -G -n "$user" | tr ' ' '\n' | sort) 2>/dev/null || true diff --git a/cdist/conf/type/__user_groups/gencode-remote b/cdist/conf/type/__user_groups/gencode-remote index 0585e90f..8120761a 100755 --- a/cdist/conf/type/__user_groups/gencode-remote +++ b/cdist/conf/type/__user_groups/gencode-remote @@ -26,15 +26,13 @@ os=$(cat "$__global/explorer/os") mkdir "$__object/files" # file has to be sorted for comparison with `comm` sort "$__object/parameter/group" > "$__object/files/group.sorted" -# Use local sort for remote groups -sort "$__object/explorer/group" > "$__object/files/group-remote.sorted" case "$state_should" in present) - changed_groups="$(comm -13 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")" + changed_groups="$(comm -13 "$__object/explorer/group" "$__object/files/group.sorted")" ;; absent) - changed_groups="$(comm -12 "$__object/files/group-remote.sorted" "$__object/files/group.sorted")" + changed_groups="$(comm -12 "$__object/explorer/group" "$__object/files/group.sorted")" ;; esac diff --git a/cdist/config.py b/cdist/config.py index 638fdf0e..9b3076e5 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -41,6 +41,7 @@ import cdist.hostsource import cdist.exec.local import cdist.exec.remote import cdist.util.ipaddr as ipaddr +import cdist.util.python_type_util as pytype_util import cdist.configuration @@ -91,13 +92,15 @@ class Config: shutil.rmtree(path) def __init__(self, local, remote, dry_run=False, jobs=None, - cleanup_cmds=None, remove_remote_files_dirs=False): + cleanup_cmds=None, remove_remote_files_dirs=False, + timestamp=False): self.local = local self.remote = remote self._open_logger() self.dry_run = dry_run self.jobs = jobs + self.timestamp = timestamp if cleanup_cmds: self.cleanup_cmds = cleanup_cmds else: @@ -437,7 +440,8 @@ class Config: cleanup_cmds.append(cleanup_cmd) c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs, cleanup_cmds=cleanup_cmds, - remove_remote_files_dirs=remove_remote_files_dirs) + remove_remote_files_dirs=remove_remote_files_dirs, + timestamp=args.timestamp) c.run() cls._remove_paths() @@ -475,6 +479,7 @@ class Config: 'dry' if self.dry_run else 'configuration') self._init_files_dirs() + self.local.collect_python_types() self.explorer.run_global_explorers(self.local.global_explorer_out_path) try: @@ -790,6 +795,21 @@ class Config: args = [param, cdist_type.name] self.log.warning(format, *args) + def _timeit(self, func, msg_prefix): + def wrapper_func(*args, **kwargs): + loglevel = self.log.getEffectiveLevel() + if loglevel >= logging.VERBOSE and self.timestamp: + start_time = time.time() + rv = func(*args, **kwargs) + end_time = time.time() + duration = end_time - start_time + self.log.verbose("%s duration: %.6f seconds", + msg_prefix, duration) + else: + rv = func(*args, **kwargs) + return rv + return wrapper_func + def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) @@ -798,7 +818,19 @@ class Config: cdist_object.name) self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: - self.manifest.run_type_manifest(cdist_object) + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( + "Running manifest and explorers for " + cdist_object.name) + self.explorer.run_type_explorers(cdist_object, + transfer_type_explorers) + if pytype_util.is_python_type(cdist_object.cdist_type): + self._timeit(self.manifest.run_py_type_manifest, + "Python type manifest for {}".format( + cdist_object.name))(cdist_object) + else: + self._timeit(self.manifest.run_type_manifest, + "Type manifest for {}".format( + cdist_object.name))(cdist_object) self.log.trace("[ORDER_DEP] Removing order dep files for %s", cdist_object) cdist_object.cleanup() @@ -816,9 +848,21 @@ class Config: # Generate self.log.debug("Generating code for %s", cdist_object.name) - cdist_object.code_local = self.code.run_gencode_local(cdist_object) - cdist_object.code_remote = self.code.run_gencode_remote( - cdist_object) + if pytype_util.is_python_type(cdist_object.cdist_type): + cdist_object.code_local = '' + cdist_object.code_remote = self._timeit( + self.code.run_py, + "Python type generate code for {}".format( + cdist_object.name))(cdist_object) + else: + cdist_object.code_local = self._timeit( + self.code.run_gencode_local, + "Type generate code local for {}".format( + cdist_object.name))(cdist_object) + cdist_object.code_remote = self._timeit( + self.code.run_gencode_remote, + "Type generate code remote for {}".format( + cdist_object.name))(cdist_object) if cdist_object.code_local or cdist_object.code_remote: cdist_object.changed = True @@ -829,12 +873,16 @@ class Config: if cdist_object.code_local: self.log.trace("Executing local code for %s", cdist_object.name) - self.code.run_code_local(cdist_object) + self._timeit(self.code.run_code_local, + "Type run code local for {}".format( + cdist_object.name))(cdist_object) if cdist_object.code_remote: self.log.trace("Executing remote code for %s", cdist_object.name) self.code.transfer_code_remote(cdist_object) - self.code.run_code_remote(cdist_object) + self._timeit(self.code.run_code_remote, + "Type run code remote for {}".format( + cdist_object.name))(cdist_object) # Mark this object as done self.log.trace("Finishing run of %s", cdist_object.name) diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index b79cdb21..de2fbbbf 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest from cdist.core.code import Code from cdist.core.util import listdir from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val +import cdist.core.pytypes diff --git a/cdist/core/code.py b/cdist/core/code.py index 12888ba4..95a0b587 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -22,6 +22,8 @@ # import os +import inspect +from cdist.core.pytypes import get_pytype_class from . import util @@ -120,6 +122,27 @@ class Code: self.env['__cdist_log_server_socket'] = os.environ[ '__cdist_log_server_socket_export'] + def run_py(self, cdist_object): + cdist_type = cdist_object.cdist_type + type_class = get_pytype_class(cdist_type) + if type_class is not None: + env = os.environ.copy() + env.update(self.env) + message_prefix = cdist_object.name + type_obj = type_class(env=env, cdist_object=cdist_object, + local=self.local, remote=self.remote, + message_prefix=message_prefix) + if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run): + if self.local.save_output_streams: + which = 'gencode-py' + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'a+') as stderr, \ + open(stdout_path, 'a+') as stdout: + return type_obj.run(stdout=stdout, stderr=stderr) + else: + return type_obj.run() + def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type gencode_attr = getattr(cdist_type, 'gencode_{}_path'.format(which)) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 09e74dac..9e0b6dd9 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -22,9 +22,12 @@ import logging import os - +import inspect import cdist +import cdist.emulator from . import util +from cdist.core.pytypes import Command, get_pytype_class + ''' common: @@ -96,9 +99,6 @@ class Manifest: """ - ORDER_DEP_STATE_NAME = 'order_dep_state' - TYPEORDER_DEP_NAME = 'typeorder_dep' - def __init__(self, target_host, local, dry_run=False): self.target_host = target_host self.local = local @@ -225,5 +225,58 @@ class Manifest: os.remove(os.path.join(self.local.base_path, fname)) except FileNotFoundError: pass - _rm_file(Manifest.ORDER_DEP_STATE_NAME) - _rm_file(Manifest.TYPEORDER_DEP_NAME) + _rm_file(cdist.ORDER_DEP_STATE_NAME) + _rm_file(cdist.TYPEORDER_DEP_NAME) + + def env_py_type_manifest(self, cdist_object): + env = os.environ.copy() + env.update(self.env) + env.update({ + '__cdist_object_marker': self.local.object_marker_name, + '__cdist_manifest': cdist_object.cdist_type, + '__manifest': self.local.manifest_path, + '__object': cdist_object.absolute_path, + '__object_id': cdist_object.object_id, + '__object_name': cdist_object.name, + '__type': cdist_object.cdist_type.absolute_path, + }) + + return env + + def run_py_type_manifest(self, cdist_object): + cdist_type = cdist_object.cdist_type + type_class = get_pytype_class(cdist_type) + if type_class is not None: + self.log.verbose("Running python type manifest for object %s", + cdist_object.name) + message_prefix = cdist_object.name + env = self.env_py_type_manifest(cdist_object) + type_obj = type_class(env=env, cdist_object=cdist_object, + local=self.local, remote=None, + message_prefix=message_prefix) + if self.local.save_output_streams: + which = 'manifest' + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'a+') as stderr, \ + open(stdout_path, 'a+') as stdout: + self._process_py_type_manifest_entries( + type_obj, env, stdout=stdout, stderr=stderr) + else: + self._process_py_type_manifest_entries(type_obj, env) + + def _process_py_type_manifest_entries(self, type_obj, env, stdout=None, + stderr=None): + if hasattr(type_obj, 'manifest') and \ + inspect.ismethod(type_obj.manifest): + for cmd in type_obj.manifest(stdout=stdout, stderr=stderr): + if not isinstance(cmd, Command): + raise TypeError("Manifest command must be of type Command") + kwargs = { + 'argv': cmd.cmd_line(), + 'env': env, + } + if cmd.stdin: + kwargs['stdin'] = cmd.stdin + emulator = cdist.emulator.Emulator(**kwargs) + emulator.run() diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py new file mode 100644 index 00000000..9dfc8df9 --- /dev/null +++ b/cdist/core/pytypes.py @@ -0,0 +1,190 @@ +import logging +import os +import io +import sys +import re +from cdist import message, Error +import importlib.util +import inspect +import cdist + + +__all__ = ["PythonType", "Command", "command"] + + +class PythonType: + def __init__(self, env, cdist_object, local, remote, message_prefix=None): + self.env = env + self.cdist_object = cdist_object + self.local = local + self.remote = remote + if self.cdist_object: + self.object_id = cdist_object.object_id + self.object_name = cdist_object.name + self.cdist_type = cdist_object.cdist_type + self.object_path = cdist_object.absolute_path + self.explorer_path = os.path.join(self.object_path, 'explorer') + self.type_path = cdist_object.cdist_type.absolute_path + self.parameters = cdist_object.parameters + self.stdin_path = os.path.join(self.object_path, 'stdin') + if self.local: + self.log = logging.getLogger( + self.local.target_host[0] + ':' + self.object_name) + + self.message_prefix = message_prefix + self.message = None + + def get_parameter(self, name): + return self.parameters.get(name) + + def get_explorer_file(self, name): + path = os.path.join(self.explorer_path, name) + return path + + def get_explorer(self, name): + path = self.get_explorer_file(name) + with open(path, 'r') as f: + value = f.read() + if value: + value = value.strip() + return value + + def run_local(self, command, env=None): + rv = self.local.run(command, env=env, return_output=True) + if rv: + rv = rv.rstrip('\n') + return rv + + def run_remote(self, command, env=None): + rv = self.remote.run(command, env=env, return_output=True) + if rv: + rv = rv.rstrip('\n') + return rv + + def transfer(self, source, destination): + self.remote.transfer(source, destination) + + def die(self, msg): + raise Error("{}: {}".format(self.cdist_object, msg)) + + def manifest(self, stdout=None, stderr=None): + try: + if self.message_prefix: + self.message = message.Message(self.message_prefix, + self.local.messages_path) + self.env.update(self.message.env) + if stdout is not None: + stdout_save = sys.stdout + sys.stdout = stdout + if stderr is not None: + stderr_save = sys.stderr + sys.stderr = stderr + yield from self.type_manifest() + finally: + if self.message: + self.message.merge_messages() + if stdout is not None: + sys.stdout = stdout_save + if stderr is not None: + sys.stderr = stderr_save + + def run(self, stdout=None, stderr=None): + try: + if self.message_prefix: + self.message = message.Message(self.message_prefix, + self.local.messages_path) + if stdout is not None: + stdout_save = sys.stdout + sys.stdout = stdout + if stderr is not None: + stderr_save = sys.stderr + sys.stderr = stderr + return self.type_gencode() + finally: + if self.message: + self.message.merge_messages() + if stdout is not None: + sys.stdout = stdout_save + if stderr is not None: + sys.stderr = stderr_save + + def send_message(self, msg): + if self.message: + with open(self.message.messages_out, 'a') as f: + print(msg, file=f) + + def receive_message(self, pattern): + if self.message: + with open(self.message.messages_in, 'r') as f: + for line in f: + match = re.search(pattern, line) + if match: + return match + return None + + def get_args_parser(self): + pass + + def type_manifest(self): + pass + + def type_gencode(self): + pass + + +class Command: + def __init__(self, name, *args, **kwargs): + self.name = name + self.args = args + self.kwargs = kwargs + self.stdin = None + + def feed_stdin(self, value): + # If file-like object then read its value. + if value is not None and isinstance(value, io.IOBase): + value = value.read() + + # Convert to bytes file-like object. + if value is None: + self.stdin = None + elif isinstance(value, str): + self.stdin = io.BytesIO(value.encode('utf-8')) + elif isinstance(value, bytes) or isinstance(value, bytearray): + self.stdin = io.BytesIO(value) + else: + raise TypeError("value must be str, bytes, bytearray, file-like " + "object or None") + return self + + def cmd_line(self): + argv = [self.name, ] + for param in self.args: + argv.append(param) + for key, value in self.kwargs.items(): + argv.append("--{}".format(key)) + argv.append(value) + return argv + + +def command(name, *args, **kwargs): + return Command(name, *args, **kwargs) + + +def get_pytype_class(cdist_type): + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + type_class = None + if os.path.isfile(file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error( + "Only one python type class is supported, but at least" + " two found: {}".format((type_class, cl, ))) + else: + type_class = cl + return type_class diff --git a/cdist/emulator.py b/cdist/emulator.py index c55b47d2..db46c659 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -30,7 +30,9 @@ import re import cdist from cdist import core from cdist import flock -from cdist.core.manifest import Manifest +import cdist.util.python_type_util as pytype_util +from cdist.core.pytypes import get_pytype_class +import inspect class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -85,9 +87,9 @@ class Emulator: self.typeorder_path = os.path.join(self.global_path, "typeorder") self.typeorder_dep_path = os.path.join(self.global_path, - Manifest.TYPEORDER_DEP_NAME) + cdist.TYPEORDER_DEP_NAME) self.order_dep_state_path = os.path.join(self.global_path, - Manifest.ORDER_DEP_STATE_NAME) + cdist.ORDER_DEP_STATE_NAME) self.type_name = os.path.basename(argv[0]) self.cdist_type = core.CdistType(self.type_base_path, self.type_name) @@ -97,7 +99,28 @@ class Emulator: def run(self): """Emulate type commands (i.e. __file and co)""" - self.commandline() + args_parser = None + if pytype_util.is_python_type(self.cdist_type): + type_class = get_pytype_class(self.cdist_type) + if type_class is not None: + # We only need to call parse_args so we need plain instance. + type_obj = type_class(env=None, cdist_object=None, local=None, + remote=None) + if (hasattr(type_obj, 'get_args_parser') and + inspect.ismethod(type_obj.get_args_parser)): + args_parser = type_obj.get_args_parser() + self.log.trace("Using python type argument parser") + print("Using python type argument parser") + if args_parser is None: + # Fallback to classic way. + args_parser = self.get_args_parser() + self.log.trace("Fallback to classic argument parser") + print("Fallback to classic argument parser") + else: + args_parser = self.get_args_parser() + self.log.trace("Using emulator classic argument parser") + print("Using emulator classic argument parser") + self.commandline(args_parser) self.init_object() # locking for parallel execution @@ -133,9 +156,7 @@ class Emulator: colored_log = self.env.get('__cdist_colored_log', 'false') cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true' - def commandline(self): - """Parse command line""" - + def get_args_parser(self): parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS) @@ -167,8 +188,10 @@ class Emulator: # If not singleton support one positional parameter if not self.cdist_type.is_singleton: parser.add_argument("object_id", nargs=1) + return parser - # And finally parse/verify parameter + def commandline(self, parser): + """Parse command line""" self.args = parser.parse_args(self.argv[1:]) self.log.trace('Args: %s', self.args) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 370336ad..8d0ebc54 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -35,6 +35,9 @@ import cdist import cdist.message from cdist import core import cdist.exec.util as util +import cdist.util.python_type_util as pytype_util +from cdist.core import pytypes +import functools CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] @@ -369,3 +372,16 @@ class Local: raise cdist.Error( "Linking emulator from {} to {} failed: {}".format( src, dst, e.__str__())) + + def collect_python_types(self): + for cdist_type in core.CdistType.list_types(self.type_path): + if pytype_util.is_python_type(cdist_type): + self.log.trace("Detected python type %s, collecting it".format( + cdist_type.name)) + f = functools.partial(pytypes.command, cdist_type.name) + if cdist_type.name.startswith('__'): + attr_name = cdist_type.name.replace('__', '', 1) + else: + attr_name = cdist_type.name + setattr(pytypes, attr_name, f) + pytypes.__all__.append(attr_name) diff --git a/cdist/integration.py b/cdist/integration.py index 04470ea7..17b65f09 100644 --- a/cdist/integration.py +++ b/cdist/integration.py @@ -84,7 +84,7 @@ def _process_hosts_simple(action, host, manifest, verbose, """ if isinstance(host, str): hosts = [host, ] - elif isinstance(host, collections.abc.Iterable): + elif isinstance(host, collections.Iterable): hosts = host else: raise cdist.Error('Invalid host argument: {}'.format(host)) diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index f1f750ee..b8daa50f 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -126,6 +126,12 @@ class Debian: help="suite used for debootstrap, " "by default '{}'".format(defargs.suite), dest='suite', default=defargs.suite) + parser.add_argument( + '-t', '--trigger-command', + help=("trigger command that will be added to cdist config; " + "'__cdist_preos_trigger http ...' type is appended to " + "initial manifest"), + dest='trigger_command') parser.add_argument( '-y', '--remote-copy', help=("remote copy that cdist config will use, by default " diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code index d836848c..4b59d0e0 100755 --- a/cdist/preos/debootstrap/files/code +++ b/cdist/preos/debootstrap/files/code @@ -127,6 +127,13 @@ then exit 1 fi + if [ "${trigger_command}" ] + then + trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n" + else + trigger_line="" + fi + if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ] then i="$((keyfile_cnt - 1))" @@ -174,7 +181,7 @@ then fi grub_lines="${grub_manifest_line}${grub_kern_params_line}" - printf "${ssh_auth_keys_line}${grub_lines}" \ + printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \ | cat "${manifest}" - |\ cdist config \ ${cdist_params} -i - \ diff --git a/cdist/trigger.py b/cdist/trigger.py new file mode 100644 index 00000000..58f98dca --- /dev/null +++ b/cdist/trigger.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2016 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import ipaddress +import logging +import re +import socket +import http.server +import os +import socketserver +import shutil + +import cdist.config +import cdist.log +import cdist.util.ipaddr as ipaddr + + +class Trigger(): + """cdist trigger handling""" + + # Arguments that are only trigger specific + triggers_args = ["http_port", "ipv6", "directory", "source", ] + + def __init__(self, http_port=None, dry_run=False, ipv6=False, + directory=None, source=None, cdistargs=None): + self.dry_run = dry_run + self.http_port = int(http_port) + self.ipv6 = ipv6 + self.args = cdistargs + + self.directory = directory + self.source = source + + log.debug("IPv6: %s", self.ipv6) + + def run_httpd(self): + server_address = ('', self.http_port) + + if self.ipv6: + httpdcls = HTTPServerV6 + else: + httpdcls = HTTPServerV4 + httpd = httpdcls(self.args, self.directory, self.source, + server_address, TriggerHttp) + + log.debug("Starting server at port %d", self.http_port) + if self.dry_run: + log.debug("Running in dry run mode") + httpd.serve_forever() + + def run(self): + if self.http_port: + self.run_httpd() + + @classmethod + def commandline(cls, args): + global log + + # remove root logger default cdist handler and configure trigger's own + logging.getLogger().handlers = [] + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') + + log = logging.getLogger("trigger") + ownargs = {} + for targ in cls.triggers_args: + arg = getattr(args, targ) + ownargs[targ] = arg + + del arg + + t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs) + t.run() + + +class TriggerHttp(http.server.BaseHTTPRequestHandler): + actions = { + "cdist": ["config", "install", ], + "file": ["present", "absent", ], + } + + def do_HEAD(self): + self.dispatch_request() + + def do_POST(self): + self.dispatch_request() + + def do_GET(self): + self.dispatch_request() + + def _actions_regex(self): + regex = ["^/(?P", ] + regex.extend("|".join(self.actions.keys())) + regex.append(")/(?P") + regex.extend("|".join("|".join(self.actions[x]) for x in self.actions)) + regex.append(")/") + + return "".join(regex) + + def dispatch_request(self): + host = self.client_address[0] + code = 200 + message = None + + self.cdistargs = self.server.cdistargs + + actions_regex = self._actions_regex() + m = re.match(actions_regex, self.path) + + if m: + subsystem = m.group('subsystem') + action = m.group('action') + handler = getattr(self, "handler_" + subsystem) + + if action not in self.actions[subsystem]: + code = 404 + else: + code = 404 + + if code == 200: + log.debug("Calling {} -> {}".format(subsystem, action)) + try: + handler(action, host) + except cdist.Error as e: + # cdist is not broken, cdist run is broken + code = 599 # use arbitrary unassigned error code + message = str(e) + except Exception as e: + # cdist/trigger server is broken + log.exception(e) + code = 500 + + self.send_response(code=code, message=message) + self.end_headers() + + def handler_file(self, action, host): + if not self.server.directory or not self.server.source: + log.info("Cannot serve file request: directory or source " + "not setup") + return + + try: + ipaddress.ip_address(host) + except ValueError: + log.error("Host is not a valid IP address - aborting") + return + + dst = os.path.join(self.server.directory, host) + + if action == "present": + shutil.copyfile(self.server.source, dst) + if action == "absent": + if os.path.exists(dst): + os.remove(dst) + + def handler_cdist(self, action, host): + log.debug("Running cdist action %s for %s", action, host) + + if self.server.dry_run: + log.info("Dry run, skipping cdist execution") + return + + cname = action.title() + module = getattr(cdist, action) + theclass = getattr(module, cname) + + if hasattr(self.cdistargs, 'out_path'): + out_path = self.cdistargs.out_path + else: + out_path = None + host_base_path, hostdir = theclass.create_host_base_dirs( + host, theclass.create_base_root_path(out_path)) + theclass.construct_remote_exec_copy_patterns(self.cdistargs) + host_tags = None + host_name = ipaddr.resolve_target_host_name(host) + log.debug('Resolved target host name: %s', host_name) + if host_name: + target_host = host_name + else: + target_host = host + log.debug('Using target_host: %s', target_host) + log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ", + target_host, host_tags, host_base_path, hostdir, + self.cdistargs) + cfg = cdist.configuration.Configuration(self.cdistargs) + configuration = cfg.get_config(section='GLOBAL') + theclass.onehost(target_host, host_tags, host_base_path, hostdir, + self.cdistargs, parallel=False, + configuration=configuration) + + +class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer): + """ + Server that listens to both IPv4 and IPv6 requests. + """ + address_family = socket.AF_INET6 + + def __init__(self, cdistargs, directory, source, *args, **kwargs): + self.cdistargs = cdistargs + self.dry_run = cdistargs.dry_run + self.directory = directory + self.source = source + + http.server.HTTPServer.__init__(self, *args, **kwargs) + + +class HTTPServerV4(HTTPServerV6): + """ + Server that listens to IPv4 requests. + """ + address_family = socket.AF_INET diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 6bf935e8..09e9cc19 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -33,7 +33,7 @@ class AbsolutePathRequiredError(cdist.Error): return 'Absolute path required, got: {}'.format(self.path) -class FileList(collections.abc.MutableSequence): +class FileList(collections.MutableSequence): """A list that stores it's state in a file. """ @@ -102,7 +102,7 @@ class FileList(collections.abc.MutableSequence): self.__write(lines) -class DirectoryDict(collections.abc.MutableMapping): +class DirectoryDict(collections.MutableMapping): """A dict that stores it's items as files in a directory. """ diff --git a/cdist/util/python_type_util.py b/cdist/util/python_type_util.py new file mode 100644 index 00000000..fd97684e --- /dev/null +++ b/cdist/util/python_type_util.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# 2019 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + + +def is_python_type(cdist_type): + init_path = os.path.join(cdist_type.absolute_path, '__init__.py') + return os.path.exists(init_path) diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index cdac7f29..ed5c6a55 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -6,7 +6,7 @@ _cdist() prev="${COMP_WORDS[COMP_CWORD-1]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}" opts="-h --help -q --quiet -v --verbose -V --version" - cmds="banner config install inventory preos shell" + cmds="banner config install inventory preos shell trigger" case "${prevprev}" in shell) @@ -80,6 +80,14 @@ _cdist() COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + trigger) + opts="-h --help -d --debug -v --verbose -b --beta \ + -C --cache-path-pattern -c --conf-dir -i --initial-manifest \ + -j --jobs -n --dry-run -o --out-dir --remote-copy \ + --remote-exec -6 --ipv6 -H --http-port -D --directory -S --source" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; inventory) cmds="list add-host del-host add-tag del-tag" opts="-h --help -q --quiet -v --verbose" diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 1bd275ef..5d653095 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -11,7 +11,7 @@ _cdist() case $state in opts_cmds) - _arguments '1:Options and commands:(banner config install inventory preos shell -h --help -q --quiet -v --verbose -V --version)' + _arguments '1:Options and commands:(banner config install inventory preos shell trigger -h --help -q --quiet -v --verbose -V --version)' ;; *) case $words[2] in diff --git a/docs/changelog b/docs/changelog index fa741365..4de6c75b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,34 +2,8 @@ Changelog --------- next: - * Type __timezone: Add support for OpenWRT (Nico Schottelius) - -7.0.1: - * Core: Remove double definition of scan parser (Nico Schottelius) - * Type __apt_mark: Narrow down grep for hold packages (marcoduif) - * Type __apt_source: Set required options variable (Mark Verboom) - * Type __letsencrypt_cert: Update python version (Michelle) - * Explorer os_version: Add support for Daedalus (Michelle) - * Explorer machine_type: Correct incorrect VMM matching (Mark Verboom) - -7.0.0: 2022-07-31 - * Explorer machine_type: Rewrite (Dennis Camera) - * New type: __sed (Ander Punnar) - * New type: __haproxy_dualstack (Evilham and ungleich) - * Type __apt_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_update_index: Fix complaint about suite change (Matthias Stecher) - * Type __package_upgrade_all: Add new --apt-with-new-pkgs argument (Evilham) - * Type __apt_source: Fix complaint about suite change (Matthias Stecher) - * Type __package_apt: Fix complaint about suite change (Matthias Stecher) - * Type __debconf_set_selections: Fix bug where --file was unsupported (Evilham) - * Types __letsencrypt_cert, __grafana_dashboard: Improve bullseye support (Evilham) - * Type __ssh_authorized_key: Also remove tmpfile if removing line (Mark Verboom) - * Type __apt_pin: Add default priority, add comment in generated files (Daniel Fancsali) - * Type __file: make file uploading and attribute changes more atomic (Steven Armstrong) - * Type __dot_file: Add support for using --file parameter (Stephan Leemburg) - * Type __apt_ppa: Replace custom "remove-apt-repository" with add-apt-repository -r (Romain Dartigues) - * Type __apt_source: Add signed-by parameter (Daniel Fancsali) - * Explorer: add support for checkpoint (Stephan Leemburg) + * Core: Add trigger functionality (Nico Schottelius, Darko Poljak) + * Core: Implement core support for python types (Darko Poljak) 6.9.8: 2021-08-24 * Type __rsync: Rewrite (Ander Punnar) diff --git a/docs/dev/python-types/benchmark b/docs/dev/python-types/benchmark new file mode 100644 index 00000000..8fcf69f3 --- /dev/null +++ b/docs/dev/python-types/benchmark @@ -0,0 +1,376 @@ +# sh type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run +INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds + +# sh type, files exist at remote but content changes +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run +INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds + +# py type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run +INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds + +# py type, files exist at remote but content changes +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run +INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds + + + +# init test file content +head -c 102400 /dev/random > /tmp/test.file + +# sh type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run +INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds + +# sh type, files exist at remote +echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run +INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds + +# py type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run +INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds + +# py type, files exist at remote +echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run +INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds + + +# Summary + +# sh type, no file at remote +INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds +# py type, no file at remote +INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds + +# sh type, files exist at remote but content changes +INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds +# py type, files exist at remote but content changes +INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds + + +# sh type, no file at remote +INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds +# py type, no file at remote +INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds + +# sh type, files exist at remote +INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds +# py type, files exist at remote +INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds diff --git a/docs/dev/python-types/benchmark.sh b/docs/dev/python-types/benchmark.sh new file mode 100755 index 00000000..3e01941f --- /dev/null +++ b/docs/dev/python-types/benchmark.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 + +if [ $# -eq 0 ] +then + N=1 +else + N=$1 +fi + +manifest() { + bytes=$(echo "$1 * 1024" | bc) + echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root" +} + +verbosity="-vv" #"-vvv" +i=0 +while [ "$i" -lt "$N" ] +do + if [ "$N" -ne 1 ] + then + printf "iteration %d\\n" "$i" + fi + printf "shinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foo$i.bin;' + manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "pyinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foo$i.bin;' + manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "shinit present state...\\n" + manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "pyinit present state...\\n" + manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + i=$((i + 1)) +done diff --git a/docs/dev/python-types/conf/manifest/pyinit b/docs/dev/python-types/conf/manifest/pyinit new file mode 100644 index 00000000..53f15a97 --- /dev/null +++ b/docs/dev/python-types/conf/manifest/pyinit @@ -0,0 +1,7 @@ +#for x in 1; do +# echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root; +#done +#__dummy_config_py test1 + +echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root +__dummy_config_py test1 diff --git a/docs/dev/python-types/conf/manifest/shinit b/docs/dev/python-types/conf/manifest/shinit new file mode 100644 index 00000000..44129546 --- /dev/null +++ b/docs/dev/python-types/conf/manifest/shinit @@ -0,0 +1,7 @@ +#for x in 1; do +# echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root; +#done +#__dummy_config_sh test1 + +echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root +__dummy_config_sh test1 diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py new file mode 100644 index 00000000..0b5fa692 --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py @@ -0,0 +1,22 @@ +import os +import sys +from cdist.core.pytypes import * + + +class DummyConfig(PythonType): + def type_manifest(self): + print('dummy manifest stdout') + print('dummy manifest stderr\n', file=sys.stderr) + yield file_py('/root/dummy1.conf', + mode='0640', + owner='root', + group='root', + source='-').feed_stdin('dummy=1\n') + + self_path = os.path.dirname(os.path.realpath(__file__)) + conf_path = os.path.join(self_path, 'files', 'dummy.conf') + yield file_py('/root/dummy2.conf', + mode='0640', + owner='root', + group='root', + source=conf_path) diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf new file mode 100644 index 00000000..972d11ca --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf @@ -0,0 +1 @@ +dummy=2 diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf new file mode 100644 index 00000000..972d11ca --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf @@ -0,0 +1 @@ +dummy=2 diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/manifest b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest new file mode 100644 index 00000000..d675d6a3 --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest @@ -0,0 +1,6 @@ +printf 'dummy manifest stdout\n' +printf 'dummy manifest stderr\n' >&2 + +printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source - + +__file /root/dummy2.conf --mode 0600 --owner root --group root --source "$__type/files/dummy.conf" diff --git a/docs/dev/python-types/test.sh b/docs/dev/python-types/test.sh new file mode 100755 index 00000000..941ea264 --- /dev/null +++ b/docs/dev/python-types/test.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 +env | grep CDIST_PATH + +for streams in ' ' '-S' +do + for x in sh py + do + printf "[%s] Removing old foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*' + printf "[%s] Listing foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'ls foobar* dummy*' + printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams" + printf -- "----------------\\n" + ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST} + printf "[%s] Listing foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'ls foobar* dummy*' + ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST} + done +done diff --git a/docs/dev/python-types/timeit.sh b/docs/dev/python-types/timeit.sh new file mode 100755 index 00000000..f1b6c9fb --- /dev/null +++ b/docs/dev/python-types/timeit.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 + +if [ $# -eq 0 ] +then + N=1 +else + N=$1 +fi + +i=0 +while [ "$i" -lt "$N" ] +do + if [ "$N" -ne 1 ] + then + printf "iteration %d\\n" "$i" + fi + printf "shinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;' + + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST} + printf "pyinit clean state...\\n" + ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;' + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST} + + printf "shinit present state...\\n" + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST} + + printf "pyinit present state...\\n" + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST} + i=$((i + 1)) +done diff --git a/docs/dev/release-process.org b/docs/dev/release-process.org deleted file mode 100644 index 42b4f5c5..00000000 --- a/docs/dev/release-process.org +++ /dev/null @@ -1,90 +0,0 @@ -* Install requirements (Alpine) - - apk add py3-pycodestyle shellcheck py3-sphinx py3-sphinx_rtd_theme \ - py3-build twine -* Ensure your gpg setup works with the email used in the git commit! - - For me this is nico@nico-notebook.schottelius.org - - Signature / id is on nb2 -* Create ~/.pypirc -[distutils] - index-servers = - pypi - cdist - -[pypi] - username = __token__ - password = ... - -[cdist] - repository = https://upload.pypi.org/legacy/ - username = __token__ - password = ... - -* Add date in docs/changelog -* Run ./bin/cdist-build-helper -* TODO Move to "build" - - python3 -m build -* DONE git tag: when? -CLOSED: [2022-07-31 Sun 23:58] -** Asked during release process: ok -* DONE Pypi error with distutils: do not use distutils anymore -CLOSED: [2022-07-31 Sun 23:58] -python3 setup.py sdist upload -... -Creating tar archive -removing 'cdist-7.0.0' (and everything under it) -running upload -Submitting dist/cdist-7.0.0.tar.gz to https://upload.pypi.org/legacy/ -Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. -error: Upload failed (400): Invalid value for blake2_256_digest. Error: Use a valid, hex-encoded, BLAKE2 message digest. -(venv2) [22:50] nb2:cdist% - -* DONE Pypi error with twine: fixed in twine 4.0.1 -CLOSED: [2022-07-31 Sun 23:58] - -Seeing: - -(venv2) [22:47] nb2:cdist% twine upload dist/cdist-7.0.0* -Uploading distributions to https://upload.pypi.org/legacy/ -Traceback (most recent call last): - File "/usr/bin/twine", line 8, in - sys.exit(main()) - File "/usr/lib/python3.10/site-packages/twine/__main__.py", line 28, in main - result = cli.dispatch(sys.argv[1:]) - File "/usr/lib/python3.10/site-packages/twine/cli.py", line 68, in dispatch - return main(args.args) - File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 197, in main - return upload(upload_settings, parsed_args.dists) - File "/usr/lib/python3.10/site-packages/twine/commands/upload.py", line 141, in upload - resp = repository.upload(package) - File "/usr/lib/python3.10/site-packages/twine/repository.py", line 189, in upload - resp = self._upload(package) - File "/usr/lib/python3.10/site-packages/twine/repository.py", line 144, in _upload - data = package.metadata_dictionary() - File "/usr/lib/python3.10/site-packages/twine/package.py", line 181, in metadata_dictionary - "dynamic": meta.dynamic, -AttributeError: 'Wheel' object has no attribute 'dynamic' - - -Fix: - - -(venv2) [23:43] nb2:cdist% pipx run twine upload dist/* -⚠️ twine is already on your PATH and installed at /home/nico/venv2/bin/twine. Downloading and running anyway. -Uploading distributions to https://upload.pypi.org/legacy/ -Uploading cdist-7.0.0-py3-none-any.whl -100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 868.6/868.6 kB • 00:04 • 221.3 kB/s -Uploading cdist-7.0.0.tar.gz -100% ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.5/1.5 MB • 00:08 • 169.3 kB/s - -View at: -https://pypi.org/project/cdist/7.0.0/ -* TODO cdist web - - on staticweb-2022 - - Should be moved to sftp/k8s - - - Manual steps: - - ~/bin/permissions.public html/ - rsync -a html/ staticweb.ungleich.ch:/home/services/www/nico/www.cdi.st/www/manual/7.0.0/ - ssh staticweb.ungleich.ch "cd /home/services/www/nico/www.cdi.st/www/manual; ln -sf 7.0.0 latest" diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst index 9570bcfc..ad88b277 100644 --- a/docs/src/cdist-preos.rst +++ b/docs/src/cdist-preos.rst @@ -25,15 +25,16 @@ For example, to create an ubuntu PreOS: .. code-block:: sh $ cdist preos ubuntu /preos/preos-ubuntu -B -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu + -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ + -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/" For more info about the available options see the cdist manual page. -This will bootstrap (``-B``) ubuntu PreOS in the ``/preos/preos-ubuntu`` -directory, it will be configured (``-C``) using default the built-in initial -manifest and with specified ssh authorized key (``-k``). -After bootstrapping and configuration, the PXE boot directory will be -created (``-p``) in ``/preos/pxe-ubuntu``. +This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it +will be configured (``-C``) using default built-in initial manifest and with +specified ssh authorized key (``-k``) and with specified trigger command (``-t``). +After bootstrapping and configuration PXE +boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``. After PreOS is created, new machines can be booted using the created PXE (after proper dhcp and tftp settings). @@ -41,8 +42,17 @@ After PreOS is created, new machines can be booted using the created PXE Since PreOS is configured with ssh authorized key it can be accessed through ssh, i.e. it can be further installed and configured with cdist. -Implementing a new PreOS sub-command ------------------------------------- +When installing and configuring new machines using cdist's PreOS concept +cdist can use triggering for host installation/configuration, which is described +in the previous chapter. + +When new machine is booted with PreOS then trigger command is executed. +Machine will connect to cdist trigger server. If the request is, for example, +for installation then cdist trigger server will start install command for the +client host using parameters specified at trigger server startup. + +Implementing new PreOS sub-command +---------------------------------- preos command is implemented as a plugin system. This plugin system scans for preos subcommands in the ``cdist/preos/`` distribution directory and also in ``~/.cdist/preos/`` directory if it exists. @@ -127,3 +137,32 @@ When you try to run this new preos you will get: In the ``commandline`` function/method you have all the freedom to actually create a PreOS. + +Simple tipical use case for using PreOS and trigger +--------------------------------------------------- +Tipical use case for using PreOS and trigger command include the following steps. + +#. Create PreOS PXE with ssh key and trigger command for installation. + + .. code-block:: sh + + $ cdist preos ubuntu /preos/ubuntu -b -C \ + -k ~/.ssh/id_rsa.pub -p /preos/pxe \ + -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/" + +#. Configure dhcp server and tftp server. + +#. On cdist host (192.168.111.5 from above) start trigger command (it will use + default init manifest for installation). + + .. code-block:: sh + + $ cdist trigger -b -v + +#. After all is set up start new machines (PXE boot). + +#. New machine boots and executes trigger command, i.e. triggers installation. + +#. Cdist trigger server starts installing host that has triggered it. + +#. After cdist install is finished new host is installed. diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst new file mode 100644 index 00000000..93e9d920 --- /dev/null +++ b/docs/src/cdist-trigger.rst @@ -0,0 +1,35 @@ +Trigger +======= + +Description +----------- +cdist supports triggering for host installation/configuration using trigger command. +This command starts trigger server at management node, for example: + +.. code-block:: sh + + $ cdist trigger -b -v + +This will start cdist trigger server in verbose mode. cdist trigger server accepts +simple requests for configuration and for installation: + +* :strong:`/cdist/install/` for installation +* :strong:`/cdist/config/` for configuration. + +Other configuration parameters are the same as in like cdist config (See `cdist `_). + +Machines can then trigger cdist trigger server with appropriate requests. +If the request is, for example, for installation (:strong:`/cdist/install/`) +then cdist trigger server will start install command for the client host using +parameters specified at trigger server startup. For the above example that means +that client will be installed using default initial manifest. + +When triggered cdist will try to reverse DNS lookup for host name and if +host name is dervied then it is used for running cdist config. If no +host name is resolved then IP address is used. + +This command returns the following response codes to client requests: + +* 200 for success +* 599 for cdist run errors +* 500 for cdist/server errors. diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 582c0938..388c3caf 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -522,3 +522,92 @@ How to include a type into upstream cdist If you think your type may be useful for others, ensure it works with the current master branch of cdist and have a look at `cdist hacking `_ on how to submit it. + + +Python types +------------ +From version/branch **beta** cdist support python types, types that are written +in python language with cdist's core support. cdist detects such type if type is +detectable as a python package, i.e. if **__init__.py** file is present in type's +root directory. Upon that detection cdist will try to run such type as core python +type. + +Note that this differs from plain cdist type where scripts are written in pure +python and have a proper shebang. + +Core python types replace manifest and gencode scripts. Parameters, singleton, +nonparallel are still defined as for common types. Explorer code is also written +in shell, since this is the code that is directly executed at target host. + +When writing python type you can extend **cdist.core.pytypes.PythonType** class. +You need to implement the following methods: + +* **get_args_parser**: implementation should return **argparse.ArgumentParser** and if + it is undefined or returned None then cdist falls back to classic type parameter + definition and argument parsing +* **type_manifest**: implementation should yield **cdist.core.pytypes.** + attribute function call result, or **yield from ()** if type does not use other types +* **type_gencode**: implementation should return a string consisting of lines + of shell code that will be executed at target host. + +**cdist.core.pytypes.** attributes correspond to detected python types. +**Note** that double underscore ('__') at the beginning of type name is removed. + +Example: + +.. code-block:: sh + + import os + import sys + from cdist.core.pytypes import * + + + class DummyConfig(PythonType): + def type_manifest(self): + print('dummy manifest stdout') + print('dummy manifest stderr\n', file=sys.stderr) + yield file_py('/root/dummy1.conf', + mode='0640', + owner='root', + group='root', + source='-').feed_stdin('dummy=1\n') + + self_path = os.path.dirname(os.path.realpath(__file__)) + conf_path = os.path.join(self_path, 'files', 'dummy.conf') + yield file_py('/root/dummy2.conf', + mode='0640', + owner='root', + group='root', + source=conf_path) + +**cdist.core.PythonType** class provides the following methods: + +* **get_parameter**: get type parameter +* **get_explorer_file**: get path to file for specified explorer +* **get_explorer**: get value for specified explorer +* **run_local**: run specified command locally +* **run_remote**: run specified command remotely +* **transfer**: transfer specified source to the remote +* **die**: raise error +* **send_message**: send message +* **receive_message**: get message. + +When running python type, cdist will save output streams to **gencode-py**, +stdout and stderr output files. + +As a reference implementation you can take a look at **__file_py** type, +which is re-implementation of **__file** type. + +Furthermore, under **docs/dev/python-types** there are sample cdist conf directory, +init manifests and scripts for running and measuring duration of samples. +There, under **conf/type/__dummy_config** you can find another example of +python type, which (unlike **__file_py** type) also uses new manifest implementation +that yields **cdist.core.pytypes.** attribute function call results. + +**NOTE** that python types implementation is under the beta, not directly controled by +the **-b/--beta** option. It is controled by the explicit usage of python types in +your config. + +Also, this documenation is only an introduction, and not a complete guide to python +types. Currently, it is just a short introduction so one can start to write and use +python types. diff --git a/docs/src/index.rst b/docs/src/index.rst index 369d5309..831eab1d 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -33,6 +33,7 @@ It natively supports IPv6 since the first release. cdist-messaging cdist-parallelization cdist-inventory + cdist-trigger cdist-preos cdist-scan cdist-integration diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 599ec3b7..89a74078 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,7 +11,7 @@ SYNOPSIS :: - cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info} ... + cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info,trigger} ... cdist banner [-h] [-l LOGLEVEL] [-q] [-v] @@ -66,21 +66,24 @@ SYNOPSIS [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist preos [preos-options] devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist preos [preos-options] ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors WHEN] [-s SHELL] @@ -88,6 +91,12 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] + [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] + [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] + [-S] [-D DIRECTORY] [-H HTTP_PORT] [--ipv6] [-O SOURCE] cdist scan -I INTERFACE [--m MODE] [--name-mapper PATH_TO_SCRIPT] [--list] [-d CONFIG_DELAY] [-t TRIGGER_DELAY] @@ -535,6 +544,10 @@ PREOS DEBIAN/DEVUAN **-s SUITE, --suite SUITE** suite used for debootstrap, by default 'stable' +**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND** + trigger command that will be added to cdist config; + '``__cdist_preos_trigger http ...``' type is appended to initial manifest + **-y REMOTE_COPY, --remote-copy REMOTE_COPY** remote copy that cdist config will use, by default internal script is used @@ -595,6 +608,10 @@ PREOS UBUNTU **-s SUITE, --suite SUITE** suite used for debootstrap, by default 'xenial' +**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND** + trigger command that will be added to cdist config; + '``__cdist_preos_trigger http ...``' type is appended to initial manifest + **-y REMOTE_COPY, --remote-copy REMOTE_COPY** remote copy that cdist config will use, by default internal script is used @@ -644,6 +661,95 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +TRIGGER +------- +Start trigger (simple http server) that waits for connections. When host +connects then it triggers config or install command and then cdist +config/install is executed which configures/installs host. +When triggered cdist will try to reverse DNS lookup for host name and if +host name is dervied then it is used for running cdist config. If no +host name is resolved then IP address is used. +Request path recognizes following requests: + +* :strong:`/cdist/config/.*` for config +* :strong:`/cdist/install/.*` for install. + +This command returns the following response codes to client requests: + +* 200 for success +* 599 for cdist run errors +* 500 for cdist/server errors. + +**-4, -force-ipv4** + Force to use IPv4 addresses only. No influence for + custom remote commands. + +**-6, --force-ipv6** + Force to use IPv6 addresses only. No influence for + custom remote commands. + +**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** + Specify custom cache path pattern. If it is not set + then default hostdir is used. + +**-c CONF_DIR, --conf-dir CONF_DIR** + Add configuration directory (can be repeated, last one + wins). + +**-D DIRECTORY, --directory DIRECTORY** + Where to create local files + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-H HTTP_PORT, --http-port HTTP_PORT** + Create trigger listener via http on specified port + +**-i MANIFEST, --initial-manifest MANIFEST** + Path to a cdist manifest or '-' to read from stdin. + +**--ipv6** + Listen to both IPv4 and IPv6 (instead of only IPv4) + +**-j [JOBS], --jobs [JOBS]** + Operate in parallel in specified maximum number of + jobs. Global explorers, object prepare and object run + are supported. Without argument CPU count is used by + default. Currently in beta. + +**-n, --dry-run** + Do not execute code. + +**-O SOURCE, --source SOURCE** + Which file to copy for creation + +**-o OUT_PATH, --out-dir OUT_PATH** + Directory to save cdist output in. + +**-P, --timestamp** + Timestamp log messages with the current local date and + time in the format: YYYYMMDDHHMMSS.us. + +**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]** + Operate by using archiving with compression where + appropriate. Supported values are: tar - tar archive, + tgz - gzip tar archive (the default), tbz2 - bzip2 tar + archive and txz - lzma tar archive. Currently in beta. + +**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** + Directory to save cdist output in on the target host. + +**--remote-copy REMOTE_COPY** + Command to use for remote copy (should behave like + scp). + +**--remote-exec REMOTE_EXEC** + Command to use for remote execution (should behave + like ssh). + +**-S, --disable-saving-output-streams** + Disable saving output streams. + SCAN ---- @@ -867,20 +973,28 @@ EXAMPLES # Configure all hosts from inventory db $ cdist config -b -A - # Create default debian PreOS in debug mode + # Create default debian PreOS in debug mode with config + # trigger command $ cdist preos debian /preos/preos-debian -vvvv -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian + -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian \ + -t "/usr/bin/curl 192.168.111.5:3000/config/" - # Create ubuntu PreOS + # Create ubuntu PreOS with install trigger command $ cdist preos ubuntu /preos/preos-ubuntu -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu + -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" - # Create ubuntu PreOS on drive /dev/sdb + # Create ubuntu PreOS on drive /dev/sdb with install trigger command # and set root password to 'password'. $ cdist preos ubuntu /mnt -B -C \ -k ~/.ssh/id_rsa.pub -D /dev/sdb \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" \ -P password + # Start trigger in verbose mode that will configure host using specified + # init manifest + % cdist trigger -v -i ~/.cdist/manifest/init-for-triggered + ENVIRONMENT -----------