From 9a2e5758f59b27fa17799bd67ba5accf82168f38 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 26 Jan 2019 17:00:03 +0100 Subject: [PATCH 01/14] Implement triggering functionality --- cdist/argparse.py | 24 +- .../type/__cdist_preos_trigger/gencode-remote | 12 + cdist/conf/type/__cdist_preos_trigger/man.rst | 45 ++++ .../conf/type/__cdist_preos_trigger/manifest | 67 ++++++ .../__cdist_preos_trigger/parameter/required | 1 + cdist/preos/debootstrap/debootstrap.py | 6 + cdist/preos/debootstrap/files/code | 9 +- cdist/trigger.py | 225 ++++++++++++++++++ completions/bash/cdist-completion.bash | 10 +- completions/zsh/_cdist | 2 +- docs/changelog | 3 + docs/src/cdist-preos.rst | 55 ++++- docs/src/cdist-trigger.rst | 33 +++ docs/src/index.rst | 1 + docs/src/man1/cdist.rst | 121 +++++++++- 15 files changed, 593 insertions(+), 21 deletions(-) create mode 100644 cdist/conf/type/__cdist_preos_trigger/gencode-remote create mode 100644 cdist/conf/type/__cdist_preos_trigger/man.rst create mode 100644 cdist/conf/type/__cdist_preos_trigger/manifest create mode 100644 cdist/conf/type/__cdist_preos_trigger/parameter/required create mode 100644 cdist/trigger.py create mode 100644 docs/src/cdist-trigger.rst diff --git a/cdist/argparse.py b/cdist/argparse.py index 611c484a..47320e01 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -5,12 +5,13 @@ import logging import collections import functools import cdist.configuration +import cdist.trigger import cdist.preos import cdist.info # set of beta sub-commands -BETA_COMMANDS = set(('install', 'inventory', )) +BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', )) # set of beta arguments for sub-commands BETA_ARGS = { 'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )), @@ -468,6 +469,27 @@ def get_parsers(): 'pattern', nargs='?', help='Glob pattern.') parser['info'].set_defaults(func=cdist.info.Info.commandline) + # Trigger + parser['trigger'] = parser['sub'].add_parser( + 'trigger', parents=[parser['loglevel'], + parser['beta'], + parser['config_main']]) + parser['trigger'].add_argument( + '-6', '--ipv6', default=False, + help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), + action='store_true') + parser['trigger'].add_argument( + '-D', '--directory', action='store', required=False, + help=('Where to create local files')) + parser['trigger'].add_argument( + '-H', '--http-port', action='store', default=3000, required=False, + help=('Create trigger listener via http on specified port')) + parser['trigger'].add_argument( + '-S', '--source', action='store', required=False, + help=('Which file to copy for creation')) + + parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline) + for p in parser: parser[p].epilog = EPILOG diff --git a/cdist/conf/type/__cdist_preos_trigger/gencode-remote b/cdist/conf/type/__cdist_preos_trigger/gencode-remote new file mode 100644 index 00000000..d5e9fe5c --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/gencode-remote @@ -0,0 +1,12 @@ +#!/bin/sh + +os=$(cat "$__global/explorer/os") + +case "$os" in + devuan) + echo "update-rc.d cdist-preos-trigger defaults > /dev/null" + ;; + *) + ;; +esac + diff --git a/cdist/conf/type/__cdist_preos_trigger/man.rst b/cdist/conf/type/__cdist_preos_trigger/man.rst new file mode 100644 index 00000000..abbd553b --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/man.rst @@ -0,0 +1,45 @@ +cdist-type__cdist_preos_trigger(7) +================================== + +NAME +---- +cdist-type__cdist_preos_trigger - configure cdist preos trigger + + +DESCRIPTION +----------- +Create cdist PreOS trigger by creating systemd unit file that will be started +at boot and will execute trigger command - connect to specified host and port. + + +REQUIRED PARAMETERS +------------------- +trigger-command + Command that will be executed as a PreOS cdist trigger. + + +OPTIONAL PARAMETERS +------------------- +None + + +EXAMPLES +-------- + +.. code-block:: sh + + # Configure default curl trigger for host cdist.ungleich.ch at port 80. + __cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80' + + +AUTHORS +------- +Darko Poljak + + +COPYING +------- +Copyright \(C) 2016 Darko Poljak. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__cdist_preos_trigger/manifest b/cdist/conf/type/__cdist_preos_trigger/manifest new file mode 100644 index 00000000..750a0548 --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/manifest @@ -0,0 +1,67 @@ +#!/bin/sh + +os="$(cat "$__global/explorer/os")" +trigger_command=$(cat "$__object/parameter/trigger-command") + +case "$os" in + devuan) + __file /etc/init.d/cdist-preos-trigger --owner root \ + --group root \ + --mode 755 \ + --source - << EOF +#!/bin/sh +# /etc/init.d/cdist-preos-trigger + +### BEGIN INIT INFO +# Provides: cdist-preos-trigger +# Required-Start: \$all +# Required-Stop: +# Default-Start: 2 3 4 5 S +# Default-Stop: 0 1 6 +# Short-Description: Execute cdist preos trigger command +# Description: Execute cdist preos trigger commnad. +### END INIT INFO + +case "\$1" in + start) + echo "Starting cdist-preos-trigger command" + ${trigger_command} & + ;; + stop) + # no-op + ;; + *) + echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}" + exit 1 + ;; +esac + +exit 0 +EOF + ;; + *) + __file /etc/systemd/system/cdist-preos-trigger.service --owner root \ + --group root \ + --mode 644 \ + --source - << EOF +[Unit] +Description=preos trigger +Wants=network-online.target +After=network.target network-online.target + +[Service] +Type=simple +Restart=no +# Broken systemd +ExecStartPre=/bin/sleep 5 +ExecStart=${trigger_command} + +[Install] +WantedBy=multi-user.target +EOF + + require="__file/etc/systemd/system/cdist-preos-trigger.service" \ + __start_on_boot cdist-preos-trigger + ;; +esac + diff --git a/cdist/conf/type/__cdist_preos_trigger/parameter/required b/cdist/conf/type/__cdist_preos_trigger/parameter/required new file mode 100644 index 00000000..3407a482 --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/parameter/required @@ -0,0 +1 @@ +trigger-command diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index f53dd4a7..7b56008d 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -127,6 +127,12 @@ class Debian(object): help="suite used for debootstrap, " "by default '{}'".format(defargs.suite), dest='suite', default=defargs.suite) + parser.add_argument( + '-t', '--trigger-command', + help=("trigger command that will be added to cdist config; " + "'__cdist_preos_trigger http ...' type is appended to " + "initial manifest"), + dest='trigger_command') parser.add_argument( '-y', '--remote-copy', help=("remote copy that cdist config will use, by default " diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code index 9e37003b..50e73972 100755 --- a/cdist/preos/debootstrap/files/code +++ b/cdist/preos/debootstrap/files/code @@ -127,6 +127,13 @@ then exit 1 fi + if [ "${trigger_command}" ] + then + trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n" + else + trigger_line="" + fi + if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ] then i="$((keyfile_cnt - 1))" @@ -174,7 +181,7 @@ then fi grub_lines="${grub_manifest_line}${grub_kern_params_line}" - printf "${ssh_auth_keys_line}${grub_lines}" \ + printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \ | cat "${manifest}" - |\ cdist config \ ${cdist_params} -i - \ diff --git a/cdist/trigger.py b/cdist/trigger.py new file mode 100644 index 00000000..4e66ac6b --- /dev/null +++ b/cdist/trigger.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2016 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import ipaddress +import logging +import re +import socket +import http.server +import os +import socketserver +import shutil + +import cdist.config +import cdist.log +import cdist.util.ipaddr as ipaddr + + +class Trigger(): + """cdist trigger handling""" + + # Arguments that are only trigger specific + triggers_args = ["http_port", "ipv6", "directory", "source", ] + + def __init__(self, http_port=None, dry_run=False, ipv6=False, + directory=None, source=None, cdistargs=None): + self.dry_run = dry_run + self.http_port = int(http_port) + self.ipv6 = ipv6 + self.args = cdistargs + + self.directory = directory + self.source = source + + log.debug("IPv6: %s", self.ipv6) + + def run_httpd(self): + server_address = ('', self.http_port) + + if self.ipv6: + httpdcls = HTTPServerV6 + else: + httpdcls = HTTPServerV4 + httpd = httpdcls(self.args, self.directory, self.source, + server_address, TriggerHttp) + + log.debug("Starting server at port %d", self.http_port) + if self.dry_run: + log.debug("Running in dry run mode") + httpd.serve_forever() + + def run(self): + if self.http_port: + self.run_httpd() + + @classmethod + def commandline(cls, args): + global log + + # remove root logger default cdist handler and configure trigger's own + logging.getLogger().handlers = [] + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') + + log = logging.getLogger("trigger") + ownargs = {} + for targ in cls.triggers_args: + arg = getattr(args, targ) + ownargs[targ] = arg + + del arg + + t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs) + t.run() + + +class TriggerHttp(http.server.BaseHTTPRequestHandler): + actions = { + "cdist": ["config", "install", ], + "file": ["present", "absent", ], + } + + def do_HEAD(self): + self.dispatch_request() + + def do_POST(self): + self.dispatch_request() + + def do_GET(self): + self.dispatch_request() + + def _actions_regex(self): + regex = ["^/(?P", ] + regex.extend("|".join(self.actions.keys())) + regex.append(")/(?P") + regex.extend("|".join("|".join(self.actions[x]) for x in self.actions)) + regex.append(")/") + + return "".join(regex) + + def dispatch_request(self): + host = self.client_address[0] + code = 200 + message = None + + self.cdistargs = self.server.cdistargs + + actions_regex = self._actions_regex() + m = re.match(actions_regex, self.path) + + if m: + subsystem = m.group('subsystem') + action = m.group('action') + handler = getattr(self, "handler_" + subsystem) + + if action not in self.actions[subsystem]: + code = 404 + else: + code = 404 + + if code == 200: + log.debug("Calling {} -> {}".format(subsystem, action)) + try: + handler(action, host) + except cdist.Error as e: + # cdist is not broken, cdist run is broken + code = 599 # use arbitrary unassigned error code + message = str(e) + except Exception as e: + # cdist/trigger server is broken + code = 500 + + self.send_response(code=code, message=message) + self.end_headers() + + def handler_file(self, action, host): + if not self.server.directory or not self.server.source: + log.info("Cannot serve file request: directory or source " + "not setup") + return + + try: + ipaddress.ip_address(host) + except ValueError: + log.error("Host is not a valid IP address - aborting") + return + + dst = os.path.join(self.server.directory, host) + + if action == "present": + shutil.copyfile(self.server.source, dst) + if action == "absent": + if os.path.exists(dst): + os.remove(dst) + + def handler_cdist(self, action, host): + log.debug("Running cdist action %s for %s", action, host) + + if self.server.dry_run: + log.info("Dry run, skipping cdist execution") + return + + cname = action.title() + module = getattr(cdist, action) + theclass = getattr(module, cname) + + if hasattr(self.cdistargs, 'out_path'): + out_path = self.cdistargs.out_path + else: + out_path = None + host_base_path, hostdir = theclass.create_host_base_dirs( + host, theclass.create_base_root_path(out_path)) + theclass.construct_remote_exec_copy_patterns(self.cdistargs) + host_tags = None + host_name = ipaddr.resolve_target_host_name(host) + log.debug('Resolved target host name: %s', host_name) + if host_name: + target_host = host_name + else: + target_host = host + log.debug('Using target_host: %s', target_host) + log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ", + target_host, host_tags, host_base_path, hostdir, + self.cdistargs) + theclass.onehost(target_host, host_tags, host_base_path, hostdir, + self.cdistargs, parallel=False) + + +class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer): + """ + Server that listens to both IPv4 and IPv6 requests. + """ + address_family = socket.AF_INET6 + + def __init__(self, cdistargs, directory, source, *args, **kwargs): + self.cdistargs = cdistargs + self.dry_run = cdistargs.dry_run + self.directory = directory + self.source = source + + http.server.HTTPServer.__init__(self, *args, **kwargs) + + +class HTTPServerV4(HTTPServerV6): + """ + Server that listens to IPv4 requests. + """ + address_family = socket.AF_INET diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index cdac7f29..ed5c6a55 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -6,7 +6,7 @@ _cdist() prev="${COMP_WORDS[COMP_CWORD-1]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}" opts="-h --help -q --quiet -v --verbose -V --version" - cmds="banner config install inventory preos shell" + cmds="banner config install inventory preos shell trigger" case "${prevprev}" in shell) @@ -80,6 +80,14 @@ _cdist() COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + trigger) + opts="-h --help -d --debug -v --verbose -b --beta \ + -C --cache-path-pattern -c --conf-dir -i --initial-manifest \ + -j --jobs -n --dry-run -o --out-dir --remote-copy \ + --remote-exec -6 --ipv6 -H --http-port -D --directory -S --source" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; inventory) cmds="list add-host del-host add-tag del-tag" opts="-h --help -q --quiet -v --verbose" diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 1bd275ef..5d653095 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -11,7 +11,7 @@ _cdist() case $state in opts_cmds) - _arguments '1:Options and commands:(banner config install inventory preos shell -h --help -q --quiet -v --verbose -V --version)' + _arguments '1:Options and commands:(banner config install inventory preos shell trigger -h --help -q --quiet -v --verbose -V --version)' ;; *) case $words[2] in diff --git a/docs/changelog b/docs/changelog index 9f1e6245..25566aa8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Core: Add trigger functionality (Nico Schottelius, Darko Poljak) + 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) * Explorer os_release: Add fallbacks (Dennis Camera) diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst index 9570bcfc..59d0d13e 100644 --- a/docs/src/cdist-preos.rst +++ b/docs/src/cdist-preos.rst @@ -25,15 +25,16 @@ For example, to create an ubuntu PreOS: .. code-block:: sh $ cdist preos ubuntu /preos/preos-ubuntu -B -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu + -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" For more info about the available options see the cdist manual page. -This will bootstrap (``-B``) ubuntu PreOS in the ``/preos/preos-ubuntu`` -directory, it will be configured (``-C``) using default the built-in initial -manifest and with specified ssh authorized key (``-k``). -After bootstrapping and configuration, the PXE boot directory will be -created (``-p``) in ``/preos/pxe-ubuntu``. +This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it +will be configured (``-C``) using default built-in initial manifest and with +specified ssh authorized key (``-k``) and with specified trigger command (``-t``). +After bootstrapping and configuration PXE +boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``. After PreOS is created, new machines can be booted using the created PXE (after proper dhcp and tftp settings). @@ -41,8 +42,17 @@ After PreOS is created, new machines can be booted using the created PXE Since PreOS is configured with ssh authorized key it can be accessed through ssh, i.e. it can be further installed and configured with cdist. -Implementing a new PreOS sub-command ------------------------------------- +When installing and configuring new machines using cdist's PreOS concept +cdist can use triggering for host installation/configuration, which is described +in the previous chapter. + +When new machine is booted with PreOS then trigger command is executed. +Machine will connect to cdist trigger server. If the request is, for example, +for installation then cdist trigger server will start install command for the +client host using parameters specified at trigger server startup. + +Implementing new PreOS sub-command +---------------------------------- preos command is implemented as a plugin system. This plugin system scans for preos subcommands in the ``cdist/preos/`` distribution directory and also in ``~/.cdist/preos/`` directory if it exists. @@ -127,3 +137,32 @@ When you try to run this new preos you will get: In the ``commandline`` function/method you have all the freedom to actually create a PreOS. + +Simple tipical use case for using PreOS and trigger +--------------------------------------------------- +Tipical use case for using PreOS and trigger command include the following steps. + +#. Create PreOS PXE with ssh key and trigger command for installation. + + .. code-block:: sh + + $ cdist preos ubuntu /preos/ubuntu -b -C \ + -k ~/.ssh/id_rsa.pub -p /preos/pxe \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" + +#. Configure dhcp server and tftp server. + +#. On cdist host (192.168.111.5 from above) start trigger command (it will use + default init manifest for installation). + + .. code-block:: sh + + $ cdist trigger -b -v + +#. After all is set up start new machines (PXE boot). + +#. New machine boots and executes trigger command, i.e. triggers installation. + +#. Cdist trigger server starts installing host that has triggered it. + +#. After cdist install is finished new host is installed. diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst new file mode 100644 index 00000000..94477c29 --- /dev/null +++ b/docs/src/cdist-trigger.rst @@ -0,0 +1,33 @@ +Trigger +======= + +Description +----------- +cdist supports triggering for host installation/configuration using trigger command. +This command starts trigger server at management node, for example: + +.. code-block:: sh + + $ cdist trigger -b -v + +This will start cdist trigger server in verbose mode. cdist trigger server accepts +simple requests for configuration and for installation: + +* :strong:`/cdist/install/.*` for installation +* :strong:`/cdist/config/.*` for configuration. + +Machines can then trigger cdist trigger server with appropriate requests. +If the request is, for example, for installation (:strong:`/cdist/install/`) +then cdist trigger server will start install command for the client host using +parameters specified at trigger server startup. For the above example that means +that client will be installed using default initial manifest. + +When triggered cdist will try to reverse DNS lookup for host name and if +host name is dervied then it is used for running cdist config. If no +host name is resolved then IP address is used. + +This command returns the following response codes to client requests: + +* 200 for success +* 599 for cdist run errors +* 500 for cdist/server errors. diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..652a018a 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -33,6 +33,7 @@ It natively supports IPv6 since the first release. cdist-messaging cdist-parallelization cdist-inventory + cdist-trigger cdist-preos cdist-integration cdist-reference diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 38248821..fba2b322 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,7 +11,7 @@ SYNOPSIS :: - cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info} ... + cdist [-h] [-V] {banner,config,install,inventory,preos,shell,info,trigger} ... cdist banner [-h] [-l LOGLEVEL] [-q] [-v] @@ -67,27 +67,37 @@ SYNOPSIS [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist preos [preos-options] devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist preos [preos-options] ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] + [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY] + [-H HTTP_PORT] [-S SOURCE] + DESCRIPTION ----------- @@ -534,6 +544,10 @@ PREOS DEBIAN/DEVUAN **-s SUITE, --suite SUITE** suite used for debootstrap, by default 'stable' +**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND** + trigger command that will be added to cdist config; + '``__cdist_preos_trigger http ...``' type is appended to initial manifest + **-y REMOTE_COPY, --remote-copy REMOTE_COPY** remote copy that cdist config will use, by default internal script is used @@ -594,6 +608,10 @@ PREOS UBUNTU **-s SUITE, --suite SUITE** suite used for debootstrap, by default 'xenial' +**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND** + trigger command that will be added to cdist config; + '``__cdist_preos_trigger http ...``' type is appended to initial manifest + **-y REMOTE_COPY, --remote-copy REMOTE_COPY** remote copy that cdist config will use, by default internal script is used @@ -643,6 +661,83 @@ Display information for cdist (global explorers, types). **-t, --types** Display info for types. +TRIGGER +------- +Start trigger (simple http server) that waits for connections. When host +connects then it triggers config or install command and then cdist +config/install is executed which configures/installs host. +When triggered cdist will try to reverse DNS lookup for host name and if +host name is dervied then it is used for running cdist config. If no +host name is resolved then IP address is used. +Request path recognizes following requests: + +* :strong:`/cdist/config/.*` for config +* :strong:`/cdist/install/.*` for install. + +This command returns the following response codes to client requests: + +* 200 for success +* 599 for cdist run errors +* 500 for cdist/server errors. + + +**-6, --ipv6** + + Listen to both IPv4 and IPv6 (instead of only IPv4) + +**-b, --beta** + + Enable beta functionality. + +**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** + + Sepcify custom cache path pattern. It can also be set by + CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then + default hostdir is used. For more info on format see + :strong:`CACHE PATH PATTERN FORMAT` below. + +**-c CONF_DIR, --conf-dir CONF_DIR** + + Add configuration directory (can be repeated, last one wins) + +**-D DIRECTORY, --directory DIRECTORY** + Where to create local files + +**-H HTTP_PORT, --http-port HTTP_PORT** + + Create trigger listener via http on specified port + +**-i MANIFEST, --initial-manifest MANIFEST** + + path to a cdist manifest or '-' to read from stdin. + +**-j [JOBS], --jobs [JOBS]** + + Specify the maximum number of parallel jobs, currently + only global explorers are supported + +**-n, --dry-run** + + do not execute code + +**-o OUT_PATH, --out-dir OUT_PATH** + + directory to save cdist output in + +**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** + + Directory to save cdist output in on the target host + +**--remote-copy REMOTE_COPY** + + Command to use for remote copy (should behave like scp) + +**--remote-exec REMOTE_EXEC** + + Command to use for remote execution (should behave like ssh) + +**-S SOURCE, --source SOURCE** + Which file to copy for creation CONFIGURATION ------------- @@ -838,20 +933,28 @@ EXAMPLES # Configure all hosts from inventory db $ cdist config -b -A - # Create default debian PreOS in debug mode + # Create default debian PreOS in debug mode with config + # trigger command $ cdist preos debian /preos/preos-debian -vvvv -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian + -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian \ + -t "/usr/bin/curl 192.168.111.5:3000/config/" - # Create ubuntu PreOS + # Create ubuntu PreOS with install trigger command $ cdist preos ubuntu /preos/preos-ubuntu -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu + -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" - # Create ubuntu PreOS on drive /dev/sdb + # Create ubuntu PreOS on drive /dev/sdb with install trigger command # and set root password to 'password'. $ cdist preos ubuntu /mnt -B -C \ -k ~/.ssh/id_rsa.pub -D /dev/sdb \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" \ -P password + # Start trigger in verbose mode that will configure host using specified + # init manifest + % cdist trigger -v -i ~/.cdist/manifest/init-for-triggered + ENVIRONMENT ----------- From 9d030bddc15f32177854ca4a54c8018b6f492497 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Sep 2019 20:08:55 +0200 Subject: [PATCH 02/14] ++ --- cdist/argparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 47320e01..323383ae 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -11,7 +11,7 @@ import cdist.info # set of beta sub-commands -BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', )) +BETA_COMMANDS = set(('install', 'inventory', 'trigger', )) # set of beta arguments for sub-commands BETA_ARGS = { 'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )), From 912c6759034f7034d2299fc4e63e09e342b49b17 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Jan 2019 16:06:36 +0100 Subject: [PATCH 03/14] Log trigger server error --- cdist/trigger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/trigger.py b/cdist/trigger.py index 4e66ac6b..1b9a384b 100644 --- a/cdist/trigger.py +++ b/cdist/trigger.py @@ -145,6 +145,7 @@ class TriggerHttp(http.server.BaseHTTPRequestHandler): message = str(e) except Exception as e: # cdist/trigger server is broken + log.exception(e) code = 500 self.send_response(code=code, message=message) From a09467917374edc91fb3c6df3d5ea5abd2a585d1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Jan 2019 16:46:40 +0100 Subject: [PATCH 04/14] Update trigger to config --- cdist/argparse.py | 11 +++--- cdist/config.py | 13 ++++--- docs/src/man1/cdist.rst | 80 +++++++++++++++++++++++------------------ 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 323383ae..ece7ff10 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -473,11 +473,8 @@ def get_parsers(): parser['trigger'] = parser['sub'].add_parser( 'trigger', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['config_main']]) - parser['trigger'].add_argument( - '-6', '--ipv6', default=False, - help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), - action='store_true') parser['trigger'].add_argument( '-D', '--directory', action='store', required=False, help=('Where to create local files')) @@ -485,7 +482,11 @@ def get_parsers(): '-H', '--http-port', action='store', default=3000, required=False, help=('Create trigger listener via http on specified port')) parser['trigger'].add_argument( - '-S', '--source', action='store', required=False, + '--ipv6', default=False, + help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), + action='store_true') + parser['trigger'].add_argument( + '-O', '--source', action='store', required=False, help=('Which file to copy for creation')) parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline) diff --git a/cdist/config.py b/cdist/config.py index 97cc1da6..44afe7d5 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -144,7 +144,7 @@ class Config(object): # Determine forcing IPv4/IPv6 options if any, only for # default remote commands. - if args.force_ipv: + if hasattr(args, 'force_ipv') and args.force_ipv: force_addr_opt = " -{}".format(args.force_ipv) else: force_addr_opt = "" @@ -357,10 +357,13 @@ class Config(object): @staticmethod def _address_family(args): - if args.force_ipv == 4: - family = socket.AF_INET - elif args.force_ipv == 6: - family = socket.AF_INET6 + if hasattr(args, 'force_ipv'): + if args.force_ipv == 4: + family = socket.AF_INET + elif args.force_ipv == 6: + family = socket.AF_INET6 + else: + family = 0 else: family = 0 return family diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index fba2b322..01e25c4e 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -91,12 +91,12 @@ SYNOPSIS cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] - cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] - [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] - [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] - [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] - [--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY] - [-H HTTP_PORT] [-S SOURCE] + cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] + [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] + [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] + [-S] [-D DIRECTORY] [-H HTTP_PORT] [--ipv6] [-O SOURCE] DESCRIPTION @@ -680,64 +680,76 @@ This command returns the following response codes to client requests: * 599 for cdist run errors * 500 for cdist/server errors. +**-4, -force-ipv4** + Force to use IPv4 addresses only. No influence for + custom remote commands. -**-6, --ipv6** - - Listen to both IPv4 and IPv6 (instead of only IPv4) - -**-b, --beta** - - Enable beta functionality. +**-6, --force-ipv6** + Force to use IPv6 addresses only. No influence for + custom remote commands. **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** - - Sepcify custom cache path pattern. It can also be set by - CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then - default hostdir is used. For more info on format see - :strong:`CACHE PATH PATTERN FORMAT` below. + Specify custom cache path pattern. If it is not set + then default hostdir is used. **-c CONF_DIR, --conf-dir CONF_DIR** - - Add configuration directory (can be repeated, last one wins) + Add configuration directory (can be repeated, last one + wins). **-D DIRECTORY, --directory DIRECTORY** Where to create local files -**-H HTTP_PORT, --http-port HTTP_PORT** +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. +**-H HTTP_PORT, --http-port HTTP_PORT** Create trigger listener via http on specified port **-i MANIFEST, --initial-manifest MANIFEST** + Path to a cdist manifest or '-' to read from stdin. - path to a cdist manifest or '-' to read from stdin. +**--ipv6** + Listen to both IPv4 and IPv6 (instead of only IPv4) **-j [JOBS], --jobs [JOBS]** - - Specify the maximum number of parallel jobs, currently - only global explorers are supported + Operate in parallel in specified maximum number of + jobs. Global explorers, object prepare and object run + are supported. Without argument CPU count is used by + default. Currently in beta. **-n, --dry-run** + Do not execute code. - do not execute code +**-O SOURCE, --source SOURCE** + Which file to copy for creation **-o OUT_PATH, --out-dir OUT_PATH** + Directory to save cdist output in. - directory to save cdist output in +**-P, --timestamp** + Timestamp log messages with the current local date and + time in the format: YYYYMMDDHHMMSS.us. + +**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]** + Operate by using archiving with compression where + appropriate. Supported values are: tar - tar archive, + tgz - gzip tar archive (the default), tbz2 - bzip2 tar + archive and txz - lzma tar archive. Currently in beta. **-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** - - Directory to save cdist output in on the target host + Directory to save cdist output in on the target host. **--remote-copy REMOTE_COPY** - - Command to use for remote copy (should behave like scp) + Command to use for remote copy (should behave like + scp). **--remote-exec REMOTE_EXEC** + Command to use for remote execution (should behave + like ssh). - Command to use for remote execution (should behave like ssh) +**-S, --disable-saving-output-streams** + Disable saving output streams. -**-S SOURCE, --source SOURCE** - Which file to copy for creation CONFIGURATION ------------- From 1a5cc67cc8f16262806697978313f0855872eab6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Jan 2019 16:35:02 +0100 Subject: [PATCH 05/14] Add missing configuration arg --- cdist/config.py | 13 +++++-------- cdist/trigger.py | 5 ++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 44afe7d5..97cc1da6 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -144,7 +144,7 @@ class Config(object): # Determine forcing IPv4/IPv6 options if any, only for # default remote commands. - if hasattr(args, 'force_ipv') and args.force_ipv: + if args.force_ipv: force_addr_opt = " -{}".format(args.force_ipv) else: force_addr_opt = "" @@ -357,13 +357,10 @@ class Config(object): @staticmethod def _address_family(args): - if hasattr(args, 'force_ipv'): - if args.force_ipv == 4: - family = socket.AF_INET - elif args.force_ipv == 6: - family = socket.AF_INET6 - else: - family = 0 + if args.force_ipv == 4: + family = socket.AF_INET + elif args.force_ipv == 6: + family = socket.AF_INET6 else: family = 0 return family diff --git a/cdist/trigger.py b/cdist/trigger.py index 1b9a384b..58f98dca 100644 --- a/cdist/trigger.py +++ b/cdist/trigger.py @@ -200,8 +200,11 @@ class TriggerHttp(http.server.BaseHTTPRequestHandler): log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ", target_host, host_tags, host_base_path, hostdir, self.cdistargs) + cfg = cdist.configuration.Configuration(self.cdistargs) + configuration = cfg.get_config(section='GLOBAL') theclass.onehost(target_host, host_tags, host_base_path, hostdir, - self.cdistargs, parallel=False) + self.cdistargs, parallel=False, + configuration=configuration) class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer): From 4a5a475e3e79d4a1b8d50b0b96b5291840ff5bda Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 1 Feb 2019 17:49:14 +0100 Subject: [PATCH 06/14] Updated the man pages for the cdist trigger and preos --- docs/src/cdist-preos.rst | 4 ++-- docs/src/cdist-trigger.rst | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst index 59d0d13e..ad88b277 100644 --- a/docs/src/cdist-preos.rst +++ b/docs/src/cdist-preos.rst @@ -26,7 +26,7 @@ For example, to create an ubuntu PreOS: $ cdist preos ubuntu /preos/preos-ubuntu -B -C \ -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ - -t "/usr/bin/curl 192.168.111.5:3000/install/" + -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/" For more info about the available options see the cdist manual page. @@ -148,7 +148,7 @@ Tipical use case for using PreOS and trigger command include the following steps $ cdist preos ubuntu /preos/ubuntu -b -C \ -k ~/.ssh/id_rsa.pub -p /preos/pxe \ - -t "/usr/bin/curl 192.168.111.5:3000/install/" + -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/" #. Configure dhcp server and tftp server. diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst index 94477c29..93e9d920 100644 --- a/docs/src/cdist-trigger.rst +++ b/docs/src/cdist-trigger.rst @@ -13,8 +13,10 @@ This command starts trigger server at management node, for example: This will start cdist trigger server in verbose mode. cdist trigger server accepts simple requests for configuration and for installation: -* :strong:`/cdist/install/.*` for installation -* :strong:`/cdist/config/.*` for configuration. +* :strong:`/cdist/install/` for installation +* :strong:`/cdist/config/` for configuration. + +Other configuration parameters are the same as in like cdist config (See `cdist `_). Machines can then trigger cdist trigger server with appropriate requests. If the request is, for example, for installation (:strong:`/cdist/install/`) From 3dcd27015811e1e6659c3d13e885f28815a7e9a8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 1 Nov 2018 17:56:40 +0100 Subject: [PATCH 07/14] Implement python types --- cdist/conf/type/__file_py/__init__.py | 103 +++++ cdist/conf/type/__file_py/explorer/cksum | 34 ++ cdist/conf/type/__file_py/explorer/stat | 56 +++ cdist/conf/type/__file_py/explorer/type | 33 ++ .../type/__file_py/parameter/default/state | 1 + cdist/conf/type/__file_py/parameter/optional | 5 + cdist/config.py | 65 ++- cdist/core/__init__.py | 1 + cdist/core/code.py | 42 ++ cdist/core/manifest.py | 75 +++- cdist/core/pytypes.py | 162 ++++++++ cdist/exec/local.py | 16 + cdist/util/python_type_util.py | 27 ++ docs/changelog | 1 + docs/dev/python-types/benchmark | 376 ++++++++++++++++++ docs/dev/python-types/benchmark.sh | 43 ++ docs/dev/python-types/conf/manifest/pyinit | 7 + docs/dev/python-types/conf/manifest/shinit | 7 + .../conf/type/__dummy_config_py/__init__.py | 22 + .../type/__dummy_config_py/files/dummy.conf | 1 + .../type/__dummy_config_sh/files/dummy.conf | 1 + .../conf/type/__dummy_config_sh/manifest | 6 + docs/dev/python-types/test.sh | 27 ++ docs/dev/python-types/timeit.sh | 36 ++ docs/src/cdist-type.rst | 86 ++++ scripts/cdist | 2 +- 26 files changed, 1224 insertions(+), 11 deletions(-) create mode 100644 cdist/conf/type/__file_py/__init__.py create mode 100755 cdist/conf/type/__file_py/explorer/cksum create mode 100755 cdist/conf/type/__file_py/explorer/stat create mode 100755 cdist/conf/type/__file_py/explorer/type create mode 100644 cdist/conf/type/__file_py/parameter/default/state create mode 100644 cdist/conf/type/__file_py/parameter/optional create mode 100644 cdist/core/pytypes.py create mode 100644 cdist/util/python_type_util.py create mode 100644 docs/dev/python-types/benchmark create mode 100755 docs/dev/python-types/benchmark.sh create mode 100644 docs/dev/python-types/conf/manifest/pyinit create mode 100644 docs/dev/python-types/conf/manifest/shinit create mode 100644 docs/dev/python-types/conf/type/__dummy_config_py/__init__.py create mode 100644 docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf create mode 100644 docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf create mode 100644 docs/dev/python-types/conf/type/__dummy_config_sh/manifest create mode 100755 docs/dev/python-types/test.sh create mode 100755 docs/dev/python-types/timeit.sh diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file_py/__init__.py new file mode 100644 index 00000000..1212f8fd --- /dev/null +++ b/cdist/conf/type/__file_py/__init__.py @@ -0,0 +1,103 @@ +import os +import re +import sys +from cdist.core.pytypes import * + + +class FileType(PythonType): + def get_attribute(self, stat_file, attribute, value_should): + if os.path.exists(stat_file): + if re.match('[0-9]', value_should): + index = 1 + else: + index = 2 + with open(stat_file, 'r') as f: + for line in f: + if re.match(attribute + ":", line): + fields = line.split() + return fields[index] + return None + + def set_attribute(self, attribute, value_should, destination): + cmd = { + 'group': 'chgrp', + 'owner': 'chown', + 'mode': 'chmod', + } + self.send_message("{} '{}'".format(cmd[attribute], value_should)) + return "{} '{}' '{}'".format(cmd[attribute], value_should, destination) + + def type_manifest(self): + yield from () + + def type_gencode(self): + typeis = self.get_explorer('type') + state_should = self.get_parameter('state') + + if state_should == 'exists' and typeis == 'file': + return + + source = self.get_parameter('source') + if source == '-': + source = self.stdin_path + destination = '/' + self.object_id + if state_should == 'pre-exists': + if source is not None: + self.die('--source cannot be used with --state pre-exists') + if typeis == 'file': + return None + else: + self.die('File {} does not exist'.format(destination)) + + create_file = False + upload_file = False + set_attributes = False + code = [] + if state_should == 'present' or state_should == 'exists': + if source is None: + remote_stat = self.get_explorer('stat') + if not remote_stat: + create_file = True + else: + if os.path.exists(source): + if typeis == 'file': + local_cksum = self.run_local(['cksum', source, ]) + local_cksum = local_cksum.split()[0] + remote_cksum = self.get_explorer('cksum') + remote_cksum = remote_cksum.split()[0] + upload_file = local_cksum != remote_cksum + else: + upload_file = True + else: + self.die('Source {} does not exist'.format(source)) + if create_file or upload_file: + set_attributes = True + tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination) + destination_upload = self.run_remote( + ["mktemp", tempfile_template, ]) + if upload_file: + self.transfer(source, destination_upload) + code.append('rm -rf {}'.format(destination)) + code.append('mv {} {}'.format(destination_upload, destination)) + + if state_should in ('present', 'exists', 'pre-exists', ): + for attribute in ('group', 'owner', 'mode', ): + if attribute in self.parameters: + value_should = self.get_parameter(attribute) + if attribute == 'mode': + value_should = re.sub('^0', '', value_should) + stat_file = self.get_explorer_file('stat') + value_is = self.get_attribute(stat_file, attribute, + value_should) + if set_attributes or value_should != value_is: + code.append(self.set_attribute(attribute, + value_should, + destination)) + elif state_should == 'absent': + if typeis == 'file': + code.append('rm -f {}'.format(destination)) + self.send_message('remove') + else: + self.die('Unknown state {}'.format(state_should)) + + return "\n".join(code) diff --git a/cdist/conf/type/__file_py/explorer/cksum b/cdist/conf/type/__file_py/explorer/cksum new file mode 100755 index 00000000..335e4e7a --- /dev/null +++ b/cdist/conf/type/__file_py/explorer/cksum @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Retrieve the md5sum of a file to be created, if it is already existing. +# + +destination="/$__object_id" + +if [ -e "$destination" ]; then + if [ -f "$destination" ]; then + cksum < "$destination" + else + echo "NO REGULAR FILE" + fi +else + echo "NO FILE FOUND, NO CHECKSUM CALCULATED." +fi diff --git a/cdist/conf/type/__file_py/explorer/stat b/cdist/conf/type/__file_py/explorer/stat new file mode 100755 index 00000000..8a917556 --- /dev/null +++ b/cdist/conf/type/__file_py/explorer/stat @@ -0,0 +1,56 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +# nothing to work with, nothing we could do +[ -e "$destination" ] || exit 0 + +os=$("$__explorer/os") +case "$os" in + "freebsd"|"netbsd"|"openbsd") + # FIXME: should be something like this based on man page, but can not test + stat -f "type: %ST +owner: %Du %Su +group: %Dg %Sg +mode: %Op %Sp +size: %Dz +links: %Dl +" "$destination" + ;; + "macosx") + stat -f "type: %HT +owner: %Du %Su +group: %Dg %Sg +mode: %Lp %Sp +size: %Dz +links: %Dl +" "$destination" + ;; + *) + stat --printf="type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; +esac diff --git a/cdist/conf/type/__file_py/explorer/type b/cdist/conf/type/__file_py/explorer/type new file mode 100755 index 00000000..e723047c --- /dev/null +++ b/cdist/conf/type/__file_py/explorer/type @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +if [ ! -e "$destination" ]; then + echo none +elif [ -h "$destination" ]; then + echo symlink +elif [ -f "$destination" ]; then + echo file +elif [ -d "$destination" ]; then + echo directory +else + echo unknown +fi diff --git a/cdist/conf/type/__file_py/parameter/default/state b/cdist/conf/type/__file_py/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__file_py/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__file_py/parameter/optional b/cdist/conf/type/__file_py/parameter/optional new file mode 100644 index 00000000..c696d592 --- /dev/null +++ b/cdist/conf/type/__file_py/parameter/optional @@ -0,0 +1,5 @@ +state +group +mode +owner +source diff --git a/cdist/config.py b/cdist/config.py index 97cc1da6..74cd1ae2 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -38,6 +38,7 @@ import cdist.hostsource import cdist.exec.local import cdist.exec.remote import cdist.util.ipaddr as ipaddr +import cdist.util.python_type_util as pytype_util import cdist.configuration from cdist import core, inventory from cdist.util.remoteutil import inspect_ssh_mux_opts @@ -90,13 +91,15 @@ class Config(object): shutil.rmtree(path) def __init__(self, local, remote, dry_run=False, jobs=None, - cleanup_cmds=None, remove_remote_files_dirs=False): + cleanup_cmds=None, remove_remote_files_dirs=False, + timestamp=False): self.local = local self.remote = remote self._open_logger() self.dry_run = dry_run self.jobs = jobs + self.timestamp = timestamp if cleanup_cmds: self.cleanup_cmds = cleanup_cmds else: @@ -428,7 +431,8 @@ class Config(object): cleanup_cmds.append(cleanup_cmd) c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs, cleanup_cmds=cleanup_cmds, - remove_remote_files_dirs=remove_remote_files_dirs) + remove_remote_files_dirs=remove_remote_files_dirs, + timestamp=args.timestamp) c.run() cls._remove_paths() @@ -466,6 +470,7 @@ class Config(object): 'dry' if self.dry_run else 'configuration')) self._init_files_dirs() + self.local.collect_python_types() self.explorer.run_global_explorers(self.local.global_explorer_out_path) try: @@ -779,15 +784,41 @@ class Config(object): args = [param, cdist_type.name] self.log.warning(format, *args) + def _timeit(self, func, msg_prefix): + def wrapper_func(*args, **kwargs): + loglevel = self.log.getEffectiveLevel() + if loglevel >= logging.VERBOSE and self.timestamp: + start_time = time.time() + rv = func(*args, **kwargs) + end_time = time.time() + duration = end_time - start_time + self.log.verbose("%s duration: %.6f seconds", + msg_prefix, duration) + else: + rv = func(*args, **kwargs) + return rv + return wrapper_func + def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) self.log.verbose("Preparing object {}".format(cdist_object.name)) self.log.verbose( "Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: - self.manifest.run_type_manifest(cdist_object) + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( + "Running manifest and explorers for " + cdist_object.name) + self.explorer.run_type_explorers(cdist_object, + transfer_type_explorers) + if pytype_util.is_python_type(cdist_object.cdist_type): + self._timeit(self.manifest.run_py_type_manifest, + "Python type manifest for {}".format( + cdist_object.name))(cdist_object) + else: + self._timeit(self.manifest.run_type_manifest, + "Type manifest for {}".format( + cdist_object.name))(cdist_object) self.log.trace("[ORDER_DEP] Removing order dep files for %s", cdist_object) cdist_object.cleanup() @@ -805,9 +836,21 @@ class Config(object): # Generate self.log.debug("Generating code for %s" % (cdist_object.name)) - cdist_object.code_local = self.code.run_gencode_local(cdist_object) - cdist_object.code_remote = self.code.run_gencode_remote( - cdist_object) + if pytype_util.is_python_type(cdist_object.cdist_type): + cdist_object.code_local = '' + cdist_object.code_remote = self._timeit( + self.code.run_py, + "Python type generate code for {}".format( + cdist_object.name))(cdist_object) + else: + cdist_object.code_local = self._timeit( + self.code.run_gencode_local, + "Type generate code local for {}".format( + cdist_object.name))(cdist_object) + cdist_object.code_remote = self._timeit( + self.code.run_gencode_remote, + "Type generate code remote for {}".format( + cdist_object.name))(cdist_object) if cdist_object.code_local or cdist_object.code_remote: cdist_object.changed = True @@ -818,12 +861,16 @@ class Config(object): if cdist_object.code_local: self.log.trace("Executing local code for %s" % (cdist_object.name)) - self.code.run_code_local(cdist_object) + self._timeit(self.code.run_code_local, + "Type run code local for {}".format( + cdist_object.name))(cdist_object) if cdist_object.code_remote: self.log.trace("Executing remote code for %s" % (cdist_object.name)) self.code.transfer_code_remote(cdist_object) - self.code.run_code_remote(cdist_object) + self._timeit(self.code.run_code_remote, + "Type run code remote for {}".format( + cdist_object.name))(cdist_object) # Mark this object as done self.log.trace("Finishing run of " + cdist_object.name) diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index b79cdb21..de2fbbbf 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest from cdist.core.code import Code from cdist.core.util import listdir from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val +import cdist.core.pytypes diff --git a/cdist/core/code.py b/cdist/core/code.py index 1550880a..34d5bd97 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -22,6 +22,10 @@ # import os +import importlib.util +import inspect +import cdist +from cdist.core.pytypes import PythonType from . import util @@ -116,6 +120,44 @@ class Code(object): if dry_run: self.env['__cdist_dry_run'] = '1' + def run_py(self, cdist_object): + cdist_type = cdist_object.cdist_type + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + + if os.path.isfile(file_path): + spec = importlib.util.spec_from_file_location(module_name, + file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + type_class = None + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error("Only one python type class is " + "supported, but at least two " + "found: {}".format((type_class, + cl, ))) + else: + type_class = cl + env = os.environ.copy() + env.update(self.env) + message_prefix = cdist_object.name + type_obj = type_class(env=env, cdist_object=cdist_object, + local=self.local, remote=self.remote, + message_prefix=message_prefix) + if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run): + if self.local.save_output_streams: + which = 'gencode-py' + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'a+') as stderr, \ + open(stdout_path, 'a+') as stdout: + return type_obj.run(stdout=stdout, stderr=stderr) + else: + return type_obj.run() + def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type script = os.path.join(self.local.type_path, diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8aeaf860..546fba25 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -22,9 +22,13 @@ import logging import os - +import importlib.util +import inspect import cdist +import cdist.emulator from . import util +from cdist.core.pytypes import PythonType, Command + ''' common: @@ -226,3 +230,72 @@ class Manifest(object): pass _rm_file(Manifest.ORDER_DEP_STATE_NAME) _rm_file(Manifest.TYPEORDER_DEP_NAME) + + def env_py_type_manifest(self, cdist_object): + env = os.environ.copy() + env.update(self.env) + env.update({ + '__cdist_object_marker': self.local.object_marker_name, + '__cdist_manifest': cdist_object.cdist_type, + '__manifest': self.local.manifest_path, + '__object': cdist_object.absolute_path, + '__object_id': cdist_object.object_id, + '__object_name': cdist_object.name, + '__type': cdist_object.cdist_type.absolute_path, + }) + + return env + + def run_py_type_manifest(self, cdist_object): + cdist_type = cdist_object.cdist_type + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + message_prefix = cdist_object.name + if os.path.isfile(file_path): + self.log.verbose("Running python type manifest for object %s", + cdist_object.name) + spec = importlib.util.spec_from_file_location(module_name, + file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + type_class = None + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error("Only one python type class is " + "supported, but at least two " + "found: {}".format((type_class, + cl, ))) + else: + type_class = cl + env = self.env_py_type_manifest(cdist_object) + type_obj = type_class(env=env, cdist_object=cdist_object, + local=self.local, remote=None, + message_prefix=message_prefix) + if self.local.save_output_streams: + which = 'manifest' + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'a+') as stderr, \ + open(stdout_path, 'a+') as stdout: + self._process_py_type_manifest_entries( + type_obj, env, stdout=stdout, stderr=stderr) + else: + self._process_py_type_manifest_entries(type_obj, env) + + def _process_py_type_manifest_entries(self, type_obj, env, stdout=None, + stderr=None): + if hasattr(type_obj, 'manifest') and \ + inspect.ismethod(type_obj.manifest): + for cmd in type_obj.manifest(stdout=stdout, stderr=stderr): + if not isinstance(cmd, Command): + raise TypeError("Manifest command must be of type Command") + kwargs = { + 'argv': cmd.cmd_line(), + 'env': env, + } + if cmd.stdin: + kwargs['stdin'] = cmd.stdin + emulator = cdist.emulator.Emulator(**kwargs) + emulator.run() diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py new file mode 100644 index 00000000..ae4164ec --- /dev/null +++ b/cdist/core/pytypes.py @@ -0,0 +1,162 @@ +import logging +import os +import io +import sys +import re +from cdist import message, Error + + +__all__ = ["PythonType", "Command", "command"] + + +class PythonType: + def __init__(self, env, cdist_object, local, remote, message_prefix=None): + self.env = env + self.cdist_object = cdist_object + self.object_id = cdist_object.object_id + self.object_name = cdist_object.name + self.cdist_type = cdist_object.cdist_type + self.local = local + self.remote = remote + self.object_path = cdist_object.absolute_path + self.type_path = cdist_object.cdist_type.absolute_path + self.explorer_path = os.path.join(self.object_path, 'explorer') + self.parameters = cdist_object.parameters + self.stdin_path = os.path.join(self.object_path, 'stdin') + self.log = logging.getLogger( + self.local.target_host[0] + ':' + self.object_name) + + self.message_prefix = message_prefix + self.message = None + + def get_parameter(self, name): + return self.parameters.get(name) + + def get_explorer_file(self, name): + path = os.path.join(self.explorer_path, name) + return path + + def get_explorer(self, name): + path = self.get_explorer_file(name) + with open(path, 'r') as f: + value = f.read() + if value: + value = value.strip() + return value + + def run_local(self, command, env=None): + rv = self.local.run(command, env=env, return_output=True) + if rv: + rv = rv.rstrip('\n') + return rv + + def run_remote(self, command, env=None): + rv = self.remote.run(command, env=env, return_output=True) + if rv: + rv = rv.rstrip('\n') + return rv + + def transfer(self, source, destination): + self.remote.transfer(source, destination) + + def die(self, msg): + raise Error("{}: {}".format(self.cdist_object, msg)) + + def type_manifest(self): + pass + + def type_gencode(self): + pass + + def manifest(self, stdout=None, stderr=None): + try: + if self.message_prefix: + self.message = message.Message(self.message_prefix, + self.local.messages_path) + self.env.update(self.message.env) + if stdout is not None: + stdout_save = sys.stdout + sys.stdout = stdout + if stderr is not None: + stderr_save = sys.stderr + sys.stderr = stderr + yield from self.type_manifest() + finally: + if self.message: + self.message.merge_messages() + if stdout is not None: + sys.stdout = stdout_save + if stderr is not None: + sys.stderr = stderr_save + + def run(self, stdout=None, stderr=None): + try: + if self.message_prefix: + self.message = message.Message(self.message_prefix, + self.local.messages_path) + if stdout is not None: + stdout_save = sys.stdout + sys.stdout = stdout + if stderr is not None: + stderr_save = sys.stderr + sys.stderr = stderr + return self.type_gencode() + finally: + if self.message: + self.message.merge_messages() + if stdout is not None: + sys.stdout = stdout_save + if stderr is not None: + sys.stderr = stderr_save + + def send_message(self, msg): + if self.message: + with open(self.message.messages_out, 'a') as f: + print(msg, file=f) + + def receive_message(self, pattern): + if self.message: + with open(self.message.messages_in, 'r') as f: + for line in f: + match = re.search(pattern, line) + if match: + return match + return None + + +class Command: + def __init__(self, name, *args, **kwargs): + self.name = name + self.args = args + self.kwargs = kwargs + self.stdin = None + + def feed_stdin(self, value): + # If file-like object then read its value. + if value is not None and isinstance(value, io.IOBase): + value = value.read() + + # Convert to bytes file-like object. + if value is None: + self.stdin = None + elif isinstance(value, str): + self.stdin = io.BytesIO(value.encode('utf-8')) + elif isinstance(value, bytes) or isinstance(value, bytearray): + self.stdin = io.BytesIO(value) + else: + raise TypeError("value must be str, bytes, bytearray, file-like " + "object or None") + return self + + def cmd_line(self): + argv = [self.name, ] + for param in self.args: + argv.append(param) + for key, value in self.kwargs.items(): + argv.append("--{}".format(key)) + argv.append(value) + return argv + + +def command(name, *args, **kwargs): + return Command(name, *args, **kwargs) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index ad6c6e36..c1877c90 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -35,6 +35,9 @@ import cdist import cdist.message from cdist import core import cdist.exec.util as util +import cdist.util.python_type_util as pytype_util +from cdist.core import pytypes +import functools CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] @@ -370,3 +373,16 @@ class Local(object): raise cdist.Error( "Linking emulator from %s to %s failed: %s" % ( src, dst, e.__str__())) + + def collect_python_types(self): + for cdist_type in core.CdistType.list_types(self.type_path): + if pytype_util.is_python_type(cdist_type): + self.log.trace("Detected python type %s, collecting it".format( + cdist_type.name)) + f = functools.partial(pytypes.command, cdist_type.name) + if cdist_type.name.startswith('__'): + attr_name = cdist_type.name.replace('__', '', 1) + else: + attr_name = cdist_type.name + setattr(pytypes, attr_name, f) + pytypes.__all__.append(attr_name) diff --git a/cdist/util/python_type_util.py b/cdist/util/python_type_util.py new file mode 100644 index 00000000..fd97684e --- /dev/null +++ b/cdist/util/python_type_util.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# 2019 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + + +def is_python_type(cdist_type): + init_path = os.path.join(cdist_type.absolute_path, '__init__.py') + return os.path.exists(init_path) diff --git a/docs/changelog b/docs/changelog index 25566aa8..9d3b94a1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Core: Add trigger functionality (Nico Schottelius, Darko Poljak) + * Core: Implement core support for python types (Darko Poljak) 6.5.1: 2020-02-15 * Type __consul_agent: Add Debian 10 support (Nico Schottelius) diff --git a/docs/dev/python-types/benchmark b/docs/dev/python-types/benchmark new file mode 100644 index 00000000..8fcf69f3 --- /dev/null +++ b/docs/dev/python-types/benchmark @@ -0,0 +1,376 @@ +# sh type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run +INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds + +# sh type, files exist at remote but content changes +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run +INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds + +# py type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run +INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds + +# py type, files exist at remote but content changes +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run +INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds + + + +# init test file content +head -c 102400 /dev/random > /tmp/test.file + +# sh type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run +INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds + +# sh type, files exist at remote +echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run +INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds + +# py type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run +INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds + +# py type, files exist at remote +echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run +INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds + + +# Summary + +# sh type, no file at remote +INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds +# py type, no file at remote +INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds + +# sh type, files exist at remote but content changes +INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds +# py type, files exist at remote but content changes +INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds + + +# sh type, no file at remote +INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds +# py type, no file at remote +INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds + +# sh type, files exist at remote +INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds +# py type, files exist at remote +INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds diff --git a/docs/dev/python-types/benchmark.sh b/docs/dev/python-types/benchmark.sh new file mode 100755 index 00000000..3e01941f --- /dev/null +++ b/docs/dev/python-types/benchmark.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 + +if [ $# -eq 0 ] +then + N=1 +else + N=$1 +fi + +manifest() { + bytes=$(echo "$1 * 1024" | bc) + echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root" +} + +verbosity="-vv" #"-vvv" +i=0 +while [ "$i" -lt "$N" ] +do + if [ "$N" -ne 1 ] + then + printf "iteration %d\\n" "$i" + fi + printf "shinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foo$i.bin;' + manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "pyinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foo$i.bin;' + manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "shinit present state...\\n" + manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "pyinit present state...\\n" + manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + i=$((i + 1)) +done diff --git a/docs/dev/python-types/conf/manifest/pyinit b/docs/dev/python-types/conf/manifest/pyinit new file mode 100644 index 00000000..53f15a97 --- /dev/null +++ b/docs/dev/python-types/conf/manifest/pyinit @@ -0,0 +1,7 @@ +#for x in 1; do +# echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root; +#done +#__dummy_config_py test1 + +echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root +__dummy_config_py test1 diff --git a/docs/dev/python-types/conf/manifest/shinit b/docs/dev/python-types/conf/manifest/shinit new file mode 100644 index 00000000..44129546 --- /dev/null +++ b/docs/dev/python-types/conf/manifest/shinit @@ -0,0 +1,7 @@ +#for x in 1; do +# echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root; +#done +#__dummy_config_sh test1 + +echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root +__dummy_config_sh test1 diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py new file mode 100644 index 00000000..0b5fa692 --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py @@ -0,0 +1,22 @@ +import os +import sys +from cdist.core.pytypes import * + + +class DummyConfig(PythonType): + def type_manifest(self): + print('dummy manifest stdout') + print('dummy manifest stderr\n', file=sys.stderr) + yield file_py('/root/dummy1.conf', + mode='0640', + owner='root', + group='root', + source='-').feed_stdin('dummy=1\n') + + self_path = os.path.dirname(os.path.realpath(__file__)) + conf_path = os.path.join(self_path, 'files', 'dummy.conf') + yield file_py('/root/dummy2.conf', + mode='0640', + owner='root', + group='root', + source=conf_path) diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf new file mode 100644 index 00000000..972d11ca --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf @@ -0,0 +1 @@ +dummy=2 diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf new file mode 100644 index 00000000..972d11ca --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf @@ -0,0 +1 @@ +dummy=2 diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/manifest b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest new file mode 100644 index 00000000..d675d6a3 --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest @@ -0,0 +1,6 @@ +printf 'dummy manifest stdout\n' +printf 'dummy manifest stderr\n' >&2 + +printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source - + +__file /root/dummy2.conf --mode 0600 --owner root --group root --source "$__type/files/dummy.conf" diff --git a/docs/dev/python-types/test.sh b/docs/dev/python-types/test.sh new file mode 100755 index 00000000..941ea264 --- /dev/null +++ b/docs/dev/python-types/test.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 +env | grep CDIST_PATH + +for streams in ' ' '-S' +do + for x in sh py + do + printf "[%s] Removing old foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*' + printf "[%s] Listing foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'ls foobar* dummy*' + printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams" + printf -- "----------------\\n" + ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST} + printf "[%s] Listing foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'ls foobar* dummy*' + ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST} + done +done diff --git a/docs/dev/python-types/timeit.sh b/docs/dev/python-types/timeit.sh new file mode 100755 index 00000000..f1b6c9fb --- /dev/null +++ b/docs/dev/python-types/timeit.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 + +if [ $# -eq 0 ] +then + N=1 +else + N=$1 +fi + +i=0 +while [ "$i" -lt "$N" ] +do + if [ "$N" -ne 1 ] + then + printf "iteration %d\\n" "$i" + fi + printf "shinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;' + + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST} + printf "pyinit clean state...\\n" + ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;' + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST} + + printf "shinit present state...\\n" + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST} + + printf "pyinit present state...\\n" + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST} + i=$((i + 1)) +done diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 582c0938..3a4a0b13 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -522,3 +522,89 @@ How to include a type into upstream cdist If you think your type may be useful for others, ensure it works with the current master branch of cdist and have a look at `cdist hacking `_ on how to submit it. + + +Python types +------------ +From version/branch **beta** cdist support python types, types that are written +in python language with cdist's core support. cdist detects such type if type is +detectable as a python package, i.e. if **__init__.py** file is present in type's +root directory. Upon that detection cdist will try to run such type as core python +type. + +Note that this differs from plain cdist type where scripts are written in pure +python and have a proper shebang. + +Core python types replace manifest and gencode scripts. Parameters, singleton, +nonparallel are still defined as for common types. Explorer code is also written +in shell, since this is the code that is directly executed at target host. + +When writing python type you can extend **cdist.core.pytypes.PythonType** class. +You need to implement the following methods: + +* **type_manifest**: implementation should yield **cdist.core.pytypes.** + attribute function call result, or **yield from ()** if type does not use other types +* **type_gencode**: implementation should return a string consisting of lines + of shell code that will be executed at target host. + +**cdist.core.pytypes.** attributes correspond to detected python types. +**Note** that double underscore ('__') at the beginning of type name is removed. + +Example: + +.. code-block:: sh + + import os + import sys + from cdist.core.pytypes import * + + + class DummyConfig(PythonType): + def type_manifest(self): + print('dummy manifest stdout') + print('dummy manifest stderr\n', file=sys.stderr) + yield file_py('/root/dummy1.conf', + mode='0640', + owner='root', + group='root', + source='-').feed_stdin('dummy=1\n') + + self_path = os.path.dirname(os.path.realpath(__file__)) + conf_path = os.path.join(self_path, 'files', 'dummy.conf') + yield file_py('/root/dummy2.conf', + mode='0640', + owner='root', + group='root', + source=conf_path) + +**cdist.core.PythonType** class provides the following methods: + +* **get_parameter**: get type parameter +* **get_explorer_file**: get path to file for specified explorer +* **get_explorer**: get value for specified explorer +* **run_local**: run specified command locally +* **run_remote**: run specified command remotely +* **transfer**: transfer specified source to the remote +* **die**: raise error +* **send_message**: send message +* **receive_message**: get message. + +When running python type, cdist will save output streams to **gencode-py**, +stdout and stderr output files. + +As a reference implementation you can take a look at **__file_py** type, +which is re-implementation of **__file** type. + +Furthermore, under **docs/dev/python-types** there are sample cdist conf directory, +init manifests and scripts for running and measuring duration of samples. +There, under **conf/type/__dummy_config** you can find another example of +python type, which (unlike **__file_py** type) also uses new manifest implementation +that yields **cdist.core.pytypes.** attribute function call results. + +**NOTE** that python types implementation is under the beta, not directly controled by +the **-b/--beta** option. It is controled by the explicit usage of python types in +your config. + +Also, this documenation is only an introduction, and not a complete guide to python +types. Currently, it is just a short introduction so one can start to write and use +python types. diff --git a/scripts/cdist b/scripts/cdist index 7bf12c01..664504a0 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -60,7 +60,7 @@ def commandline(): if __name__ == "__main__": - cdistpythonversion = '3.2' + cdistpythonversion = '3.5' if sys.version < cdistpythonversion: print('Python >= {} is required on the source host.'.format( cdistpythonversion), file=sys.stderr) From a2243cf59edb145bb3341b9ad56b14428bc90197 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 22 Jun 2019 12:55:56 +0200 Subject: [PATCH 08/14] Add support for python type defined argument parser --- cdist/conf/type/__file_py/__init__.py | 14 ++++++- cdist/core/code.py | 25 ++--------- cdist/core/manifest.py | 25 ++--------- cdist/core/pytypes.py | 60 ++++++++++++++++++++------- cdist/emulator.py | 34 ++++++++++++--- docs/src/cdist-type.rst | 3 ++ 6 files changed, 96 insertions(+), 65 deletions(-) diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file_py/__init__.py index 1212f8fd..3397b417 100644 --- a/cdist/conf/type/__file_py/__init__.py +++ b/cdist/conf/type/__file_py/__init__.py @@ -1,7 +1,7 @@ import os import re -import sys from cdist.core.pytypes import * +import argparse class FileType(PythonType): @@ -101,3 +101,15 @@ class FileType(PythonType): self.die('Unknown state {}'.format(state_should)) return "\n".join(code) + + def get_args_parser(self): + parser = argparse.ArgumentParser(add_help=False, + argument_default=argparse.SUPPRESS) + parser.add_argument('--state', dest='state', action='store', + required=False, default='present') + for param in ('group', 'mode', 'owner', 'source'): + parser.add_argument('--' + param, dest=param, action='store', + required=False, default=None) + + parser.add_argument("object_id", nargs=1) + return parser diff --git a/cdist/core/code.py b/cdist/core/code.py index 34d5bd97..ae1fddf2 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -22,10 +22,8 @@ # import os -import importlib.util import inspect -import cdist -from cdist.core.pytypes import PythonType +from cdist.core.pytypes import get_pytype_class from . import util @@ -122,25 +120,8 @@ class Code(object): def run_py(self, cdist_object): cdist_type = cdist_object.cdist_type - module_name = cdist_type.name - file_path = os.path.join(cdist_type.absolute_path, '__init__.py') - - if os.path.isfile(file_path): - spec = importlib.util.spec_from_file_location(module_name, - file_path) - m = importlib.util.module_from_spec(spec) - spec.loader.exec_module(m) - classes = inspect.getmembers(m, inspect.isclass) - type_class = None - for _, cl in classes: - if cl != PythonType and issubclass(cl, PythonType): - if type_class: - raise cdist.Error("Only one python type class is " - "supported, but at least two " - "found: {}".format((type_class, - cl, ))) - else: - type_class = cl + type_class = get_pytype_class(cdist_type) + if type_class is not None: env = os.environ.copy() env.update(self.env) message_prefix = cdist_object.name diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 546fba25..16dec465 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -22,12 +22,11 @@ import logging import os -import importlib.util import inspect import cdist import cdist.emulator from . import util -from cdist.core.pytypes import PythonType, Command +from cdist.core.pytypes import Command, get_pytype_class ''' @@ -248,27 +247,11 @@ class Manifest(object): def run_py_type_manifest(self, cdist_object): cdist_type = cdist_object.cdist_type - module_name = cdist_type.name - file_path = os.path.join(cdist_type.absolute_path, '__init__.py') - message_prefix = cdist_object.name - if os.path.isfile(file_path): + type_class = get_pytype_class(cdist_type) + if type_class is not None: self.log.verbose("Running python type manifest for object %s", cdist_object.name) - spec = importlib.util.spec_from_file_location(module_name, - file_path) - m = importlib.util.module_from_spec(spec) - spec.loader.exec_module(m) - classes = inspect.getmembers(m, inspect.isclass) - type_class = None - for _, cl in classes: - if cl != PythonType and issubclass(cl, PythonType): - if type_class: - raise cdist.Error("Only one python type class is " - "supported, but at least two " - "found: {}".format((type_class, - cl, ))) - else: - type_class = cl + message_prefix = cdist_object.name env = self.env_py_type_manifest(cdist_object) type_obj = type_class(env=env, cdist_object=cdist_object, local=self.local, remote=None, diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py index ae4164ec..9dfc8df9 100644 --- a/cdist/core/pytypes.py +++ b/cdist/core/pytypes.py @@ -4,6 +4,9 @@ import io import sys import re from cdist import message, Error +import importlib.util +import inspect +import cdist __all__ = ["PythonType", "Command", "command"] @@ -13,18 +16,20 @@ class PythonType: def __init__(self, env, cdist_object, local, remote, message_prefix=None): self.env = env self.cdist_object = cdist_object - self.object_id = cdist_object.object_id - self.object_name = cdist_object.name - self.cdist_type = cdist_object.cdist_type self.local = local self.remote = remote - self.object_path = cdist_object.absolute_path - self.type_path = cdist_object.cdist_type.absolute_path - self.explorer_path = os.path.join(self.object_path, 'explorer') - self.parameters = cdist_object.parameters - self.stdin_path = os.path.join(self.object_path, 'stdin') - self.log = logging.getLogger( - self.local.target_host[0] + ':' + self.object_name) + if self.cdist_object: + self.object_id = cdist_object.object_id + self.object_name = cdist_object.name + self.cdist_type = cdist_object.cdist_type + self.object_path = cdist_object.absolute_path + self.explorer_path = os.path.join(self.object_path, 'explorer') + self.type_path = cdist_object.cdist_type.absolute_path + self.parameters = cdist_object.parameters + self.stdin_path = os.path.join(self.object_path, 'stdin') + if self.local: + self.log = logging.getLogger( + self.local.target_host[0] + ':' + self.object_name) self.message_prefix = message_prefix self.message = None @@ -62,12 +67,6 @@ class PythonType: def die(self, msg): raise Error("{}: {}".format(self.cdist_object, msg)) - def type_manifest(self): - pass - - def type_gencode(self): - pass - def manifest(self, stdout=None, stderr=None): try: if self.message_prefix: @@ -123,6 +122,15 @@ class PythonType: return match return None + def get_args_parser(self): + pass + + def type_manifest(self): + pass + + def type_gencode(self): + pass + class Command: def __init__(self, name, *args, **kwargs): @@ -160,3 +168,23 @@ class Command: def command(name, *args, **kwargs): return Command(name, *args, **kwargs) + + +def get_pytype_class(cdist_type): + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + type_class = None + if os.path.isfile(file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error( + "Only one python type class is supported, but at least" + " two found: {}".format((type_class, cl, ))) + else: + type_class = cl + return type_class diff --git a/cdist/emulator.py b/cdist/emulator.py index 4800e2a3..7dc24a83 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -30,6 +30,9 @@ import cdist from cdist import core from cdist import flock from cdist.core.manifest import Manifest +import cdist.util.python_type_util as pytype_util +from cdist.core.pytypes import get_pytype_class +import inspect class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -100,7 +103,28 @@ class Emulator(object): def run(self): """Emulate type commands (i.e. __file and co)""" - self.commandline() + args_parser = None + if pytype_util.is_python_type(self.cdist_type): + type_class = get_pytype_class(self.cdist_type) + if type_class is not None: + # We only need to call parse_args so we need plain instance. + type_obj = type_class(env=None, cdist_object=None, local=None, + remote=None) + if (hasattr(type_obj, 'get_args_parser') and + inspect.ismethod(type_obj.get_args_parser)): + args_parser = type_obj.get_args_parser() + self.log.trace("Using python type argument parser") + print("Using python type argument parser") + if args_parser is None: + # Fallback to classic way. + args_parser = self.get_args_parser() + self.log.trace("Fallback to classic argument parser") + print("Fallback to classic argument parser") + else: + args_parser = self.get_args_parser() + self.log.trace("Using emulator classic argument parser") + print("Using emulator classic argument parser") + self.commandline(args_parser) self.init_object() # locking for parallel execution @@ -131,9 +155,7 @@ class Emulator(object): self.log = logging.getLogger(self.target_host[0]) - def commandline(self): - """Parse command line""" - + def get_args_parser(self): parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS) @@ -165,8 +187,10 @@ class Emulator(object): # If not singleton support one positional parameter if not self.cdist_type.is_singleton: parser.add_argument("object_id", nargs=1) + return parser - # And finally parse/verify parameter + def commandline(self, parser): + """Parse command line""" self.args = parser.parse_args(self.argv[1:]) self.log.trace('Args: %s' % self.args) diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 3a4a0b13..388c3caf 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -542,6 +542,9 @@ in shell, since this is the code that is directly executed at target host. When writing python type you can extend **cdist.core.pytypes.PythonType** class. You need to implement the following methods: +* **get_args_parser**: implementation should return **argparse.ArgumentParser** and if + it is undefined or returned None then cdist falls back to classic type parameter + definition and argument parsing * **type_manifest**: implementation should yield **cdist.core.pytypes.** attribute function call result, or **yield from ()** if type does not use other types * **type_gencode**: implementation should return a string consisting of lines From 420aaaee164e8ed22ed03002db64a048a20615c5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Sep 2019 20:16:55 +0200 Subject: [PATCH 09/14] Make python file type default, mv old to __file_old --- .../type/{__file_py => __file}/__init__.py | 0 cdist/conf/type/__file/explorer/stat | 35 +------- cdist/conf/type/__file/parameter/optional | 1 - .../{__file_py => __file_old}/explorer/cksum | 0 cdist/conf/type/__file_old/explorer/stat | 88 +++++++++++++++++++ .../{__file_py => __file_old}/explorer/type | 0 .../type/{__file => __file_old}/gencode-local | 0 .../{__file => __file_old}/gencode-remote | 0 .../conf/type/{__file => __file_old}/man.rst | 0 .../parameter/default/state | 0 .../parameter/optional | 1 + cdist/conf/type/__file_py/explorer/stat | 56 ------------ 12 files changed, 90 insertions(+), 91 deletions(-) rename cdist/conf/type/{__file_py => __file}/__init__.py (100%) rename cdist/conf/type/{__file_py => __file_old}/explorer/cksum (100%) create mode 100755 cdist/conf/type/__file_old/explorer/stat rename cdist/conf/type/{__file_py => __file_old}/explorer/type (100%) rename cdist/conf/type/{__file => __file_old}/gencode-local (100%) rename cdist/conf/type/{__file => __file_old}/gencode-remote (100%) rename cdist/conf/type/{__file => __file_old}/man.rst (100%) rename cdist/conf/type/{__file_py => __file_old}/parameter/default/state (100%) rename cdist/conf/type/{__file_py => __file_old}/parameter/optional (76%) delete mode 100755 cdist/conf/type/__file_py/explorer/stat diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file/__init__.py similarity index 100% rename from cdist/conf/type/__file_py/__init__.py rename to cdist/conf/type/__file/__init__.py diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 91c8cc84..c772abcf 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -52,7 +52,6 @@ fallback() { # nothing to work with, nothing we could do [ -e "$destination" ] || exit 0 - if ! command -v stat >/dev/null then fallback @@ -69,39 +68,7 @@ group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl -" "$destination" | awk '/^type/ { print tolower($0); next } { print }' - ;; - solaris) - ls1="$( ls -ld "$destination" )" - ls2="$( ls -ldn "$destination" )" - - if [ -f "$__object/parameter/mode" ] - then mode_should="$( cat "$__object/parameter/mode" )" - fi - - # yes, it is ugly hack, but if you know better way... - if [ -z "$( find "$destination" -perm "$mode_should" )" ] - then octets=888 - else octets="$( echo "$mode_should" | sed 's/^0//' )" - fi - - case "$( echo "$ls1" | cut -c1-1 )" in - -) echo 'type: regular file' ;; - d) echo 'type: directory' ;; - esac - - echo "owner: $( echo "$ls2" \ - | awk '{print $3}' ) $( echo "$ls1" \ - | awk '{print $3}' )" - - echo "group: $( echo "$ls2" \ - | awk '{print $4}' ) $( echo "$ls1" \ - | awk '{print $4}' )" - - echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" - echo "size: $( echo "$ls1" | awk '{print $5}' )" - echo "links: $( echo "$ls1" | awk '{print $2}' )" - ;; + ;; *) # NOTE: Do not use --printf here as it is not supported by BusyBox stat. # NOTE: BusyBox's stat might not support the "-c" option, in which case diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional index 9b98352c..c696d592 100644 --- a/cdist/conf/type/__file/parameter/optional +++ b/cdist/conf/type/__file/parameter/optional @@ -3,4 +3,3 @@ group mode owner source -onchange diff --git a/cdist/conf/type/__file_py/explorer/cksum b/cdist/conf/type/__file_old/explorer/cksum similarity index 100% rename from cdist/conf/type/__file_py/explorer/cksum rename to cdist/conf/type/__file_old/explorer/cksum diff --git a/cdist/conf/type/__file_old/explorer/stat b/cdist/conf/type/__file_old/explorer/stat new file mode 100755 index 00000000..13c1c208 --- /dev/null +++ b/cdist/conf/type/__file_old/explorer/stat @@ -0,0 +1,88 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +# nothing to work with, nothing we could do +[ -e "$destination" ] || exit 0 + +os=$("$__explorer/os") +case "$os" in + "freebsd"|"netbsd"|"openbsd"|"macosx") + stat -f "type: %HT +owner: %Du %Su +group: %Dg %Sg +mode: %Lp %Sp +size: %Dz +links: %Dl +" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' + ;; + alpine) + # busybox stat + stat -c "type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" + echo "size: $( echo "$ls1" | awk '{print $5}' )" + echo "links: $( echo "$ls1" | awk '{print $2}' )" + ;; + *) + stat --printf="type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; +esac diff --git a/cdist/conf/type/__file_py/explorer/type b/cdist/conf/type/__file_old/explorer/type similarity index 100% rename from cdist/conf/type/__file_py/explorer/type rename to cdist/conf/type/__file_old/explorer/type diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file_old/gencode-local similarity index 100% rename from cdist/conf/type/__file/gencode-local rename to cdist/conf/type/__file_old/gencode-local diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file_old/gencode-remote similarity index 100% rename from cdist/conf/type/__file/gencode-remote rename to cdist/conf/type/__file_old/gencode-remote diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file_old/man.rst similarity index 100% rename from cdist/conf/type/__file/man.rst rename to cdist/conf/type/__file_old/man.rst diff --git a/cdist/conf/type/__file_py/parameter/default/state b/cdist/conf/type/__file_old/parameter/default/state similarity index 100% rename from cdist/conf/type/__file_py/parameter/default/state rename to cdist/conf/type/__file_old/parameter/default/state diff --git a/cdist/conf/type/__file_py/parameter/optional b/cdist/conf/type/__file_old/parameter/optional similarity index 76% rename from cdist/conf/type/__file_py/parameter/optional rename to cdist/conf/type/__file_old/parameter/optional index c696d592..9b98352c 100644 --- a/cdist/conf/type/__file_py/parameter/optional +++ b/cdist/conf/type/__file_old/parameter/optional @@ -3,3 +3,4 @@ group mode owner source +onchange diff --git a/cdist/conf/type/__file_py/explorer/stat b/cdist/conf/type/__file_py/explorer/stat deleted file mode 100755 index 8a917556..00000000 --- a/cdist/conf/type/__file_py/explorer/stat +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# 2013 Steven Armstrong (steven-cdist armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -destination="/$__object_id" - -# nothing to work with, nothing we could do -[ -e "$destination" ] || exit 0 - -os=$("$__explorer/os") -case "$os" in - "freebsd"|"netbsd"|"openbsd") - # FIXME: should be something like this based on man page, but can not test - stat -f "type: %ST -owner: %Du %Su -group: %Dg %Sg -mode: %Op %Sp -size: %Dz -links: %Dl -" "$destination" - ;; - "macosx") - stat -f "type: %HT -owner: %Du %Su -group: %Dg %Sg -mode: %Lp %Sp -size: %Dz -links: %Dl -" "$destination" - ;; - *) - stat --printf="type: %F -owner: %u %U -group: %g %G -mode: %a %A -size: %s -links: %h -" "$destination" - ;; -esac From 1588476a91fb620a0ac80c1254bf703623a7225c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 1 Oct 2019 08:14:24 +0200 Subject: [PATCH 10/14] Remove unnecessary files --- cdist/conf/type/__file/parameter/default/state | 1 - cdist/conf/type/__file/parameter/optional | 5 ----- 2 files changed, 6 deletions(-) delete mode 100644 cdist/conf/type/__file/parameter/default/state delete mode 100644 cdist/conf/type/__file/parameter/optional diff --git a/cdist/conf/type/__file/parameter/default/state b/cdist/conf/type/__file/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__file/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional deleted file mode 100644 index c696d592..00000000 --- a/cdist/conf/type/__file/parameter/optional +++ /dev/null @@ -1,5 +0,0 @@ -state -group -mode -owner -source From b7af305636aaba6f1b3872b36382c8fbb802a605 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 1 Oct 2019 08:15:01 +0200 Subject: [PATCH 11/14] Align with shell type: implement onchange --- cdist/conf/type/__file/__init__.py | 11 ++++++++++- cdist/conf/type/__file_py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) create mode 120000 cdist/conf/type/__file_py diff --git a/cdist/conf/type/__file/__init__.py b/cdist/conf/type/__file/__init__.py index 3397b417..bcb525fc 100644 --- a/cdist/conf/type/__file/__init__.py +++ b/cdist/conf/type/__file/__init__.py @@ -52,6 +52,7 @@ class FileType(PythonType): create_file = False upload_file = False set_attributes = False + fire_onchange = False code = [] if state_should == 'present' or state_should == 'exists': if source is None: @@ -72,6 +73,7 @@ class FileType(PythonType): self.die('Source {} does not exist'.format(source)) if create_file or upload_file: set_attributes = True + fire_onchange = True tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination) destination_upload = self.run_remote( ["mktemp", tempfile_template, ]) @@ -90,6 +92,7 @@ class FileType(PythonType): value_is = self.get_attribute(stat_file, attribute, value_should) if set_attributes or value_should != value_is: + fire_onchange = True code.append(self.set_attribute(attribute, value_should, destination)) @@ -97,9 +100,15 @@ class FileType(PythonType): if typeis == 'file': code.append('rm -f {}'.format(destination)) self.send_message('remove') + fire_onchange = True else: self.die('Unknown state {}'.format(state_should)) + if fire_onchange: + onchange = self.get_parameter('onchange') + if onchange: + code.append(onchange) + return "\n".join(code) def get_args_parser(self): @@ -107,7 +116,7 @@ class FileType(PythonType): argument_default=argparse.SUPPRESS) parser.add_argument('--state', dest='state', action='store', required=False, default='present') - for param in ('group', 'mode', 'owner', 'source'): + for param in ('group', 'mode', 'owner', 'source', 'onchange'): parser.add_argument('--' + param, dest=param, action='store', required=False, default=None) diff --git a/cdist/conf/type/__file_py b/cdist/conf/type/__file_py new file mode 120000 index 00000000..efa910bd --- /dev/null +++ b/cdist/conf/type/__file_py @@ -0,0 +1 @@ +__file \ No newline at end of file From 595cfec57233728feeafbdb3131daeffdb95b778 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 30 Nov 2019 23:29:16 +0100 Subject: [PATCH 12/14] Adapt order dependency impl to python types impl --- cdist/__init__.py | 3 +++ cdist/core/manifest.py | 7 ++----- cdist/emulator.py | 5 ++--- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index c673b3ba..279fd502 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -47,6 +47,9 @@ REMOTE_COPY = "scp -o User=root -q" REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" +ORDER_DEP_STATE_NAME = 'order_dep_state' +TYPEORDER_DEP_NAME = 'typeorder_dep' + class Error(Exception): """Base exception class for this project""" diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 16dec465..84bc813f 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -100,9 +100,6 @@ class Manifest(object): """ - ORDER_DEP_STATE_NAME = 'order_dep_state' - TYPEORDER_DEP_NAME = 'typeorder_dep' - def __init__(self, target_host, local, dry_run=False): self.target_host = target_host self.local = local @@ -227,8 +224,8 @@ class Manifest(object): os.remove(os.path.join(self.local.base_path, fname)) except FileNotFoundError: pass - _rm_file(Manifest.ORDER_DEP_STATE_NAME) - _rm_file(Manifest.TYPEORDER_DEP_NAME) + _rm_file(cdist.ORDER_DEP_STATE_NAME) + _rm_file(cdist.TYPEORDER_DEP_NAME) def env_py_type_manifest(self, cdist_object): env = os.environ.copy() diff --git a/cdist/emulator.py b/cdist/emulator.py index 7dc24a83..00060a10 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -29,7 +29,6 @@ import sys import cdist from cdist import core from cdist import flock -from cdist.core.manifest import Manifest import cdist.util.python_type_util as pytype_util from cdist.core.pytypes import get_pytype_class import inspect @@ -87,9 +86,9 @@ class Emulator(object): self.typeorder_path = os.path.join(self.global_path, "typeorder") self.typeorder_dep_path = os.path.join(self.global_path, - Manifest.TYPEORDER_DEP_NAME) + cdist.TYPEORDER_DEP_NAME) self.order_dep_state_path = os.path.join(self.global_path, - Manifest.ORDER_DEP_STATE_NAME) + cdist.ORDER_DEP_STATE_NAME) self.type_name = os.path.basename(argv[0]) self.cdist_type = core.CdistType(self.type_base_path, self.type_name) From 77d704113ce7c1065a77b9b01ab3f9954e8b80ca Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Feb 2020 14:52:10 +0100 Subject: [PATCH 13/14] Update file stat from master --- cdist/conf/type/__file/explorer/stat | 35 ++++++++++++- cdist/conf/type/__file_old/explorer/stat | 64 +++++++++++++++++------- 2 files changed, 80 insertions(+), 19 deletions(-) diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index c772abcf..91c8cc84 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -52,6 +52,7 @@ fallback() { # nothing to work with, nothing we could do [ -e "$destination" ] || exit 0 + if ! command -v stat >/dev/null then fallback @@ -68,7 +69,39 @@ group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl - ;; +" "$destination" | awk '/^type/ { print tolower($0); next } { print }' + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" + echo "size: $( echo "$ls1" | awk '{print $5}' )" + echo "links: $( echo "$ls1" | awk '{print $2}' )" + ;; *) # NOTE: Do not use --printf here as it is not supported by BusyBox stat. # NOTE: BusyBox's stat might not support the "-c" option, in which case diff --git a/cdist/conf/type/__file_old/explorer/stat b/cdist/conf/type/__file_old/explorer/stat index 13c1c208..91c8cc84 100755 --- a/cdist/conf/type/__file_old/explorer/stat +++ b/cdist/conf/type/__file_old/explorer/stat @@ -2,6 +2,7 @@ # # 2013 Steven Armstrong (steven-cdist armstrong.cc) # 2019 Nico Schottelius (nico-cdist at schottelius.org) +# 2020 Dennis Camera (dennis.camera at ssrq-sds-fds.ch) # # This file is part of cdist. # @@ -21,29 +22,54 @@ destination="/$__object_id" +fallback() { + # Fallback: Patch the output together, manually. + + ls_line=$(ls -ldn "$destination") + + uid=$(echo "$ls_line" | awk '{ print $3 }') + gid=$(echo "$ls_line" | awk '{ print $4 }') + + owner=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/passwd) + group=$(awk -F: -v uid="$uid" '$3 == uid { print $1; f=1 } END { if (!f) print "UNKNOWN" }' /etc/group) + + mode_text=$(echo "$ls_line" | awk '{ print $1 }') + mode=$(echo "$mode_text" | awk '{ k=0; for (i=0; i<=8; i++) k += ((substr($1, i+2, 1) ~ /[rwx]/) * 2^(8-i)); printf("%0o", k) }') + + size=$(echo "$ls_line" | awk '{ print $5 }') + links=$(echo "$ls_line" | awk '{ print $2 }') + + printf 'type: %s\nowner: %d %s\ngroup: %d %s\nmode: %s %s\nsize: %d\nlinks: %d\n' \ + "$("$__type_explorer/type")" \ + "$uid" "$owner" \ + "$gid" "$group" \ + "$mode" "$mode_text" \ + "$size" \ + "$links" +} + + # nothing to work with, nothing we could do [ -e "$destination" ] || exit 0 -os=$("$__explorer/os") -case "$os" in - "freebsd"|"netbsd"|"openbsd"|"macosx") + +if ! command -v stat >/dev/null +then + fallback + exit +fi + + +case $("$__explorer/os") +in + freebsd|netbsd|openbsd|macosx) stat -f "type: %HT owner: %Du %Su group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl -" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' - ;; - alpine) - # busybox stat - stat -c "type: %F -owner: %u %U -group: %g %G -mode: %a %A -size: %s -links: %h -" "$destination" +" "$destination" | awk '/^type/ { print tolower($0); next } { print }' ;; solaris) ls1="$( ls -ld "$destination" )" @@ -77,12 +103,14 @@ links: %h echo "links: $( echo "$ls1" | awk '{print $2}' )" ;; *) - stat --printf="type: %F + # NOTE: Do not use --printf here as it is not supported by BusyBox stat. + # NOTE: BusyBox's stat might not support the "-c" option, in which case + # we fall through to the shell fallback. + stat -c "type: %F owner: %u %U group: %g %G mode: %a %A size: %s -links: %h -" "$destination" - ;; +links: %h" "$destination" 2>/dev/null || fallback + ;; esac From 5b9c652221af7fa9cfba56ea75d5237bc309c805 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 20 Apr 2021 07:46:30 +0200 Subject: [PATCH 14/14] Fix merging mistake --- cdist/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/config.py b/cdist/config.py index 74e397f1..9b3076e5 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -872,7 +872,7 @@ class Config: if not self.dry_run: if cdist_object.code_local: self.log.trace("Executing local code for %s", - cdist_object.ame)) + cdist_object.name) self._timeit(self.code.run_code_local, "Type run code local for {}".format( cdist_object.name))(cdist_object)