From 9f1afb59c42f309055c7fddb71dbe35bc1e2f579 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 26 Jan 2019 17:00:03 +0100 Subject: [PATCH 01/12] Implement triggering functionality --- cdist/argparse.py | 24 +- .../type/__cdist_preos_trigger/gencode-remote | 12 + cdist/conf/type/__cdist_preos_trigger/man.rst | 45 ++++ .../conf/type/__cdist_preos_trigger/manifest | 67 ++++++ .../__cdist_preos_trigger/parameter/required | 1 + cdist/preos/debootstrap/debootstrap.py | 6 + cdist/preos/debootstrap/files/code | 9 +- cdist/trigger.py | 225 ++++++++++++++++++ completions/bash/cdist-completion.bash | 10 +- completions/zsh/_cdist | 2 +- docs/changelog | 3 + docs/src/cdist-preos.rst | 43 +++- docs/src/cdist-trigger.rst | 33 +++ docs/src/index.rst | 1 + docs/src/man1/cdist.rst | 122 +++++++++- 15 files changed, 588 insertions(+), 15 deletions(-) create mode 100644 cdist/conf/type/__cdist_preos_trigger/gencode-remote create mode 100644 cdist/conf/type/__cdist_preos_trigger/man.rst create mode 100644 cdist/conf/type/__cdist_preos_trigger/manifest create mode 100644 cdist/conf/type/__cdist_preos_trigger/parameter/required create mode 100644 cdist/trigger.py create mode 100644 docs/src/cdist-trigger.rst diff --git a/cdist/argparse.py b/cdist/argparse.py index ca69cdae..692dd278 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -5,11 +5,12 @@ import logging import collections import functools import cdist.configuration +import cdist.trigger import cdist.preos # set of beta sub-commands -BETA_COMMANDS = set(('install', 'inventory', )) +BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', )) # set of beta arguments for sub-commands BETA_ARGS = { 'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )), @@ -436,6 +437,27 @@ def get_parsers(): ' should be POSIX compatible shell.')) parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) + # Trigger + parser['trigger'] = parser['sub'].add_parser( + 'trigger', parents=[parser['loglevel'], + parser['beta'], + parser['config_main']]) + parser['trigger'].add_argument( + '-6', '--ipv6', default=False, + help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), + action='store_true') + parser['trigger'].add_argument( + '-D', '--directory', action='store', required=False, + help=('Where to create local files')) + parser['trigger'].add_argument( + '-H', '--http-port', action='store', default=3000, required=False, + help=('Create trigger listener via http on specified port')) + parser['trigger'].add_argument( + '-S', '--source', action='store', required=False, + help=('Which file to copy for creation')) + + parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline) + for p in parser: parser[p].epilog = EPILOG diff --git a/cdist/conf/type/__cdist_preos_trigger/gencode-remote b/cdist/conf/type/__cdist_preos_trigger/gencode-remote new file mode 100644 index 00000000..d5e9fe5c --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/gencode-remote @@ -0,0 +1,12 @@ +#!/bin/sh + +os=$(cat "$__global/explorer/os") + +case "$os" in + devuan) + echo "update-rc.d cdist-preos-trigger defaults > /dev/null" + ;; + *) + ;; +esac + diff --git a/cdist/conf/type/__cdist_preos_trigger/man.rst b/cdist/conf/type/__cdist_preos_trigger/man.rst new file mode 100644 index 00000000..abbd553b --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/man.rst @@ -0,0 +1,45 @@ +cdist-type__cdist_preos_trigger(7) +================================== + +NAME +---- +cdist-type__cdist_preos_trigger - configure cdist preos trigger + + +DESCRIPTION +----------- +Create cdist PreOS trigger by creating systemd unit file that will be started +at boot and will execute trigger command - connect to specified host and port. + + +REQUIRED PARAMETERS +------------------- +trigger-command + Command that will be executed as a PreOS cdist trigger. + + +OPTIONAL PARAMETERS +------------------- +None + + +EXAMPLES +-------- + +.. code-block:: sh + + # Configure default curl trigger for host cdist.ungleich.ch at port 80. + __cdist_preos_trigger http --trigger-command '/usr/bin/curl cdist.ungleich.ch:80' + + +AUTHORS +------- +Darko Poljak + + +COPYING +------- +Copyright \(C) 2016 Darko Poljak. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__cdist_preos_trigger/manifest b/cdist/conf/type/__cdist_preos_trigger/manifest new file mode 100644 index 00000000..750a0548 --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/manifest @@ -0,0 +1,67 @@ +#!/bin/sh + +os="$(cat "$__global/explorer/os")" +trigger_command=$(cat "$__object/parameter/trigger-command") + +case "$os" in + devuan) + __file /etc/init.d/cdist-preos-trigger --owner root \ + --group root \ + --mode 755 \ + --source - << EOF +#!/bin/sh +# /etc/init.d/cdist-preos-trigger + +### BEGIN INIT INFO +# Provides: cdist-preos-trigger +# Required-Start: \$all +# Required-Stop: +# Default-Start: 2 3 4 5 S +# Default-Stop: 0 1 6 +# Short-Description: Execute cdist preos trigger command +# Description: Execute cdist preos trigger commnad. +### END INIT INFO + +case "\$1" in + start) + echo "Starting cdist-preos-trigger command" + ${trigger_command} & + ;; + stop) + # no-op + ;; + *) + echo "Usage: /etc/init.d/cdist-preos-trigger {start|stop}" + exit 1 + ;; +esac + +exit 0 +EOF + ;; + *) + __file /etc/systemd/system/cdist-preos-trigger.service --owner root \ + --group root \ + --mode 644 \ + --source - << EOF +[Unit] +Description=preos trigger +Wants=network-online.target +After=network.target network-online.target + +[Service] +Type=simple +Restart=no +# Broken systemd +ExecStartPre=/bin/sleep 5 +ExecStart=${trigger_command} + +[Install] +WantedBy=multi-user.target +EOF + + require="__file/etc/systemd/system/cdist-preos-trigger.service" \ + __start_on_boot cdist-preos-trigger + ;; +esac + diff --git a/cdist/conf/type/__cdist_preos_trigger/parameter/required b/cdist/conf/type/__cdist_preos_trigger/parameter/required new file mode 100644 index 00000000..3407a482 --- /dev/null +++ b/cdist/conf/type/__cdist_preos_trigger/parameter/required @@ -0,0 +1 @@ +trigger-command diff --git a/cdist/preos/debootstrap/debootstrap.py b/cdist/preos/debootstrap/debootstrap.py index f53dd4a7..7b56008d 100644 --- a/cdist/preos/debootstrap/debootstrap.py +++ b/cdist/preos/debootstrap/debootstrap.py @@ -127,6 +127,12 @@ class Debian(object): help="suite used for debootstrap, " "by default '{}'".format(defargs.suite), dest='suite', default=defargs.suite) + parser.add_argument( + '-t', '--trigger-command', + help=("trigger command that will be added to cdist config; " + "'__cdist_preos_trigger http ...' type is appended to " + "initial manifest"), + dest='trigger_command') parser.add_argument( '-y', '--remote-copy', help=("remote copy that cdist config will use, by default " diff --git a/cdist/preos/debootstrap/files/code b/cdist/preos/debootstrap/files/code index 9e37003b..50e73972 100755 --- a/cdist/preos/debootstrap/files/code +++ b/cdist/preos/debootstrap/files/code @@ -127,6 +127,13 @@ then exit 1 fi + if [ "${trigger_command}" ] + then + trigger_line="__cdist_preos_trigger http --trigger-command '${trigger_command}'\n" + else + trigger_line="" + fi + if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ] then i="$((keyfile_cnt - 1))" @@ -174,7 +181,7 @@ then fi grub_lines="${grub_manifest_line}${grub_kern_params_line}" - printf "${ssh_auth_keys_line}${grub_lines}" \ + printf "${trigger_line}${ssh_auth_keys_line}${grub_lines}" \ | cat "${manifest}" - |\ cdist config \ ${cdist_params} -i - \ diff --git a/cdist/trigger.py b/cdist/trigger.py new file mode 100644 index 00000000..4e66ac6b --- /dev/null +++ b/cdist/trigger.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2016 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import ipaddress +import logging +import re +import socket +import http.server +import os +import socketserver +import shutil + +import cdist.config +import cdist.log +import cdist.util.ipaddr as ipaddr + + +class Trigger(): + """cdist trigger handling""" + + # Arguments that are only trigger specific + triggers_args = ["http_port", "ipv6", "directory", "source", ] + + def __init__(self, http_port=None, dry_run=False, ipv6=False, + directory=None, source=None, cdistargs=None): + self.dry_run = dry_run + self.http_port = int(http_port) + self.ipv6 = ipv6 + self.args = cdistargs + + self.directory = directory + self.source = source + + log.debug("IPv6: %s", self.ipv6) + + def run_httpd(self): + server_address = ('', self.http_port) + + if self.ipv6: + httpdcls = HTTPServerV6 + else: + httpdcls = HTTPServerV4 + httpd = httpdcls(self.args, self.directory, self.source, + server_address, TriggerHttp) + + log.debug("Starting server at port %d", self.http_port) + if self.dry_run: + log.debug("Running in dry run mode") + httpd.serve_forever() + + def run(self): + if self.http_port: + self.run_httpd() + + @classmethod + def commandline(cls, args): + global log + + # remove root logger default cdist handler and configure trigger's own + logging.getLogger().handlers = [] + logging.basicConfig(format='%(asctime)s %(levelname)s: %(message)s') + + log = logging.getLogger("trigger") + ownargs = {} + for targ in cls.triggers_args: + arg = getattr(args, targ) + ownargs[targ] = arg + + del arg + + t = cls(dry_run=args.dry_run, cdistargs=args, **ownargs) + t.run() + + +class TriggerHttp(http.server.BaseHTTPRequestHandler): + actions = { + "cdist": ["config", "install", ], + "file": ["present", "absent", ], + } + + def do_HEAD(self): + self.dispatch_request() + + def do_POST(self): + self.dispatch_request() + + def do_GET(self): + self.dispatch_request() + + def _actions_regex(self): + regex = ["^/(?P", ] + regex.extend("|".join(self.actions.keys())) + regex.append(")/(?P") + regex.extend("|".join("|".join(self.actions[x]) for x in self.actions)) + regex.append(")/") + + return "".join(regex) + + def dispatch_request(self): + host = self.client_address[0] + code = 200 + message = None + + self.cdistargs = self.server.cdistargs + + actions_regex = self._actions_regex() + m = re.match(actions_regex, self.path) + + if m: + subsystem = m.group('subsystem') + action = m.group('action') + handler = getattr(self, "handler_" + subsystem) + + if action not in self.actions[subsystem]: + code = 404 + else: + code = 404 + + if code == 200: + log.debug("Calling {} -> {}".format(subsystem, action)) + try: + handler(action, host) + except cdist.Error as e: + # cdist is not broken, cdist run is broken + code = 599 # use arbitrary unassigned error code + message = str(e) + except Exception as e: + # cdist/trigger server is broken + code = 500 + + self.send_response(code=code, message=message) + self.end_headers() + + def handler_file(self, action, host): + if not self.server.directory or not self.server.source: + log.info("Cannot serve file request: directory or source " + "not setup") + return + + try: + ipaddress.ip_address(host) + except ValueError: + log.error("Host is not a valid IP address - aborting") + return + + dst = os.path.join(self.server.directory, host) + + if action == "present": + shutil.copyfile(self.server.source, dst) + if action == "absent": + if os.path.exists(dst): + os.remove(dst) + + def handler_cdist(self, action, host): + log.debug("Running cdist action %s for %s", action, host) + + if self.server.dry_run: + log.info("Dry run, skipping cdist execution") + return + + cname = action.title() + module = getattr(cdist, action) + theclass = getattr(module, cname) + + if hasattr(self.cdistargs, 'out_path'): + out_path = self.cdistargs.out_path + else: + out_path = None + host_base_path, hostdir = theclass.create_host_base_dirs( + host, theclass.create_base_root_path(out_path)) + theclass.construct_remote_exec_copy_patterns(self.cdistargs) + host_tags = None + host_name = ipaddr.resolve_target_host_name(host) + log.debug('Resolved target host name: %s', host_name) + if host_name: + target_host = host_name + else: + target_host = host + log.debug('Using target_host: %s', target_host) + log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ", + target_host, host_tags, host_base_path, hostdir, + self.cdistargs) + theclass.onehost(target_host, host_tags, host_base_path, hostdir, + self.cdistargs, parallel=False) + + +class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer): + """ + Server that listens to both IPv4 and IPv6 requests. + """ + address_family = socket.AF_INET6 + + def __init__(self, cdistargs, directory, source, *args, **kwargs): + self.cdistargs = cdistargs + self.dry_run = cdistargs.dry_run + self.directory = directory + self.source = source + + http.server.HTTPServer.__init__(self, *args, **kwargs) + + +class HTTPServerV4(HTTPServerV6): + """ + Server that listens to IPv4 requests. + """ + address_family = socket.AF_INET diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index cdac7f29..ed5c6a55 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -6,7 +6,7 @@ _cdist() prev="${COMP_WORDS[COMP_CWORD-1]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}" opts="-h --help -q --quiet -v --verbose -V --version" - cmds="banner config install inventory preos shell" + cmds="banner config install inventory preos shell trigger" case "${prevprev}" in shell) @@ -80,6 +80,14 @@ _cdist() COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + trigger) + opts="-h --help -d --debug -v --verbose -b --beta \ + -C --cache-path-pattern -c --conf-dir -i --initial-manifest \ + -j --jobs -n --dry-run -o --out-dir --remote-copy \ + --remote-exec -6 --ipv6 -H --http-port -D --directory -S --source" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; inventory) cmds="list add-host del-host add-tag del-tag" opts="-h --help -q --quiet -v --verbose" diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 1bd275ef..5d653095 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -11,7 +11,7 @@ _cdist() case $state in opts_cmds) - _arguments '1:Options and commands:(banner config install inventory preos shell -h --help -q --quiet -v --verbose -V --version)' + _arguments '1:Options and commands:(banner config install inventory preos shell trigger -h --help -q --quiet -v --verbose -V --version)' ;; *) case $words[2] in diff --git a/docs/changelog b/docs/changelog index 2342b0b0..41213464 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Core: Add trigger functionality (Nico Schottelius, Darko Poljak) + 6.1.0: 2019-11-19 * Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera) diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst index e85af2de..7c0315b7 100644 --- a/docs/src/cdist-preos.rst +++ b/docs/src/cdist-preos.rst @@ -25,13 +25,14 @@ For example, to create ubuntu PreOS: .. code-block:: sh $ cdist preos ubuntu /preos/preos-ubuntu -B -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu + -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" For more info about available options see cdist manual page. This will bootstrap (``-B``) ubuntu PreOS in ``/preos/preos-ubuntu`` directory, it will be configured (``-C``) using default built-in initial manifest and with -specified ssh authorized key (``-k``). +specified ssh authorized key (``-k``) and with specified trigger command (``-t``). After bootstrapping and configuration PXE boot directory will be created (``-p``) in ``/preos/pxe-ubuntu``. @@ -41,6 +42,15 @@ proper dhcp, tftp setting). Since PreOS is configured with ssh authorized key it can be accessed throguh ssh, i.e. it can be further installed and configured with cdist. +When installing and configuring new machines using cdist's PreOS concept +cdist can use triggering for host installation/configuration, which is described +in the previous chapter. + +When new machine is booted with PreOS then trigger command is executed. +Machine will connect to cdist trigger server. If the request is, for example, +for installation then cdist trigger server will start install command for the +client host using parameters specified at trigger server startup. + Implementing new PreOS sub-command ---------------------------------- preos command is implemented as a plugin system. This plugin system scans for @@ -135,3 +145,32 @@ When you try to run this new preos you will get: In the ``commandline`` function/method you have all the freedom to actually create PreOS. + +Simple tipical use case for using PreOS and trigger +--------------------------------------------------- +Tipical use case for using PreOS and trigger command include the following steps. + +#. Create PreOS PXE with ssh key and trigger command for installation. + + .. code-block:: sh + + $ cdist preos ubuntu /preos/ubuntu -b -C \ + -k ~/.ssh/id_rsa.pub -p /preos/pxe \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" + +#. Configure dhcp server and tftp server. + +#. On cdist host (192.168.111.5 from above) start trigger command (it will use + default init manifest for installation). + + .. code-block:: sh + + $ cdist trigger -b -v + +#. After all is set up start new machines (PXE boot). + +#. New machine boots and executes trigger command, i.e. triggers installation. + +#. Cdist trigger server starts installing host that has triggered it. + +#. After cdist install is finished new host is installed. diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst new file mode 100644 index 00000000..94477c29 --- /dev/null +++ b/docs/src/cdist-trigger.rst @@ -0,0 +1,33 @@ +Trigger +======= + +Description +----------- +cdist supports triggering for host installation/configuration using trigger command. +This command starts trigger server at management node, for example: + +.. code-block:: sh + + $ cdist trigger -b -v + +This will start cdist trigger server in verbose mode. cdist trigger server accepts +simple requests for configuration and for installation: + +* :strong:`/cdist/install/.*` for installation +* :strong:`/cdist/config/.*` for configuration. + +Machines can then trigger cdist trigger server with appropriate requests. +If the request is, for example, for installation (:strong:`/cdist/install/`) +then cdist trigger server will start install command for the client host using +parameters specified at trigger server startup. For the above example that means +that client will be installed using default initial manifest. + +When triggered cdist will try to reverse DNS lookup for host name and if +host name is dervied then it is used for running cdist config. If no +host name is resolved then IP address is used. + +This command returns the following response codes to client requests: + +* 200 for success +* 599 for cdist run errors +* 500 for cdist/server errors. diff --git a/docs/src/index.rst b/docs/src/index.rst index 5e54d8fc..54bfb93c 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -32,6 +32,7 @@ is being used in small up to enterprise grade environments. cdist-messaging cdist-parallelization cdist-inventory + cdist-trigger cdist-preos cdist-integration cdist-reference diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 7f368e68..69b8cfc5 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,7 +11,7 @@ SYNOPSIS :: - cdist [-h] [-V] {banner,config,install,inventory,preos,shell} ... + cdist [-h] [-V] {banner,config,install,inventory,preos,shell,trigger} ... cdist banner [-h] [-l LOGLEVEL] [-q] [-v] @@ -65,25 +65,35 @@ SYNOPSIS [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist preos devuan [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist preos ubuntu [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] [-C] [-c CDIST_PARAMS] [-D DRIVE] [-e REMOTE_EXEC] [-i MANIFEST] [-k KEYFILE ] [-m MIRROR] [-P ROOT_PASSWORD] [-p PXE_BOOT_DIR] [-r] - [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] + [-S SCRIPT] [-s SUITE] [-t TRIGGER_COMMAND] + [-y REMOTE_COPY] target_dir cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] + cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] + [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY] + [-H HTTP_PORT] [-S SOURCE] + DESCRIPTION ----------- @@ -519,6 +529,10 @@ PREOS DEBIAN/DEVUAN **-s SUITE, --suite SUITE** suite used for debootstrap, by default 'stable' +**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND** + trigger command that will be added to cdist config; + '``__cdist_preos_trigger http ...``' type is appended to initial manifest + **-y REMOTE_COPY, --remote-copy REMOTE_COPY** remote copy that cdist config will use, by default internal script is used @@ -579,6 +593,10 @@ PREOS UBUNTU **-s SUITE, --suite SUITE** suite used for debootstrap, by default 'xenial' +**-t TRIGGER_COMMAND, --trigger-command TRIGGER_COMMAND** + trigger command that will be added to cdist config; + '``__cdist_preos_trigger http ...``' type is appended to initial manifest + **-y REMOTE_COPY, --remote-copy REMOTE_COPY** remote copy that cdist config will use, by default internal script is used @@ -596,6 +614,84 @@ usage. Its primary use is for debugging type parameters. be POSIX compatible shell. +TRIGGER +------- +Start trigger (simple http server) that waits for connections. When host +connects then it triggers config or install command and then cdist +config/install is executed which configures/installs host. +When triggered cdist will try to reverse DNS lookup for host name and if +host name is dervied then it is used for running cdist config. If no +host name is resolved then IP address is used. +Request path recognizes following requests: + +* :strong:`/cdist/config/.*` for config +* :strong:`/cdist/install/.*` for install. + +This command returns the following response codes to client requests: + +* 200 for success +* 599 for cdist run errors +* 500 for cdist/server errors. + + +**-6, --ipv6** + + Listen to both IPv4 and IPv6 (instead of only IPv4) + +**-b, --beta** + + Enable beta functionality. + +**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** + + Sepcify custom cache path pattern. It can also be set by + CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then + default hostdir is used. For more info on format see + :strong:`CACHE PATH PATTERN FORMAT` below. + +**-c CONF_DIR, --conf-dir CONF_DIR** + + Add configuration directory (can be repeated, last one wins) + +**-D DIRECTORY, --directory DIRECTORY** + Where to create local files + +**-H HTTP_PORT, --http-port HTTP_PORT** + + Create trigger listener via http on specified port + +**-i MANIFEST, --initial-manifest MANIFEST** + + path to a cdist manifest or '-' to read from stdin. + +**-j [JOBS], --jobs [JOBS]** + + Specify the maximum number of parallel jobs, currently + only global explorers are supported + +**-n, --dry-run** + + do not execute code + +**-o OUT_PATH, --out-dir OUT_PATH** + + directory to save cdist output in + +**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** + + Directory to save cdist output in on the target host + +**--remote-copy REMOTE_COPY** + + Command to use for remote copy (should behave like scp) + +**--remote-exec REMOTE_EXEC** + + Command to use for remote execution (should behave like ssh) + +**-S SOURCE, --source SOURCE** + Which file to copy for creation + CONFIGURATION ------------- cdist obtains configuration data from the following sources in the following @@ -790,20 +886,28 @@ EXAMPLES # Configure all hosts from inventory db $ cdist config -b -A - # Create default debian PreOS in debug mode + # Create default debian PreOS in debug mode with config + # trigger command $ cdist preos debian /preos/preos-debian -vvvv -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian + -k ~/.ssh/id_rsa.pub -p /preos/pxe-debian \ + -t "/usr/bin/curl 192.168.111.5:3000/config/" - # Create ubuntu PreOS + # Create ubuntu PreOS with install trigger command $ cdist preos ubuntu /preos/preos-ubuntu -C \ - -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu + -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" - # Create ubuntu PreOS on drive /dev/sdb + # Create ubuntu PreOS on drive /dev/sdb with install trigger command # and set root password to 'password'. $ cdist preos ubuntu /mnt -B -C \ -k ~/.ssh/id_rsa.pub -D /dev/sdb \ + -t "/usr/bin/curl 192.168.111.5:3000/install/" \ -P password + # Start trigger in verbose mode that will configure host using specified + # init manifest + % cdist trigger -v -i ~/.cdist/manifest/init-for-triggered + ENVIRONMENT ----------- From 2f5808e0b7a3f953cd0a934c082d00f4711ef337 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Sep 2019 20:08:55 +0200 Subject: [PATCH 02/12] ++ --- cdist/argparse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 692dd278..c41f4830 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -10,7 +10,7 @@ import cdist.preos # set of beta sub-commands -BETA_COMMANDS = set(('install', 'inventory', 'preos', 'trigger', )) +BETA_COMMANDS = set(('install', 'inventory', 'trigger', )) # set of beta arguments for sub-commands BETA_ARGS = { 'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )), From 9b53aada1fbbfce0c7d59b2e63eb6f21273144c1 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Jan 2019 16:06:36 +0100 Subject: [PATCH 03/12] Log trigger server error --- cdist/trigger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/trigger.py b/cdist/trigger.py index 4e66ac6b..1b9a384b 100644 --- a/cdist/trigger.py +++ b/cdist/trigger.py @@ -145,6 +145,7 @@ class TriggerHttp(http.server.BaseHTTPRequestHandler): message = str(e) except Exception as e: # cdist/trigger server is broken + log.exception(e) code = 500 self.send_response(code=code, message=message) From 00059df63d88b6e33e7aedf2e077bc0268407825 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Jan 2019 16:46:40 +0100 Subject: [PATCH 04/12] Update trigger to config --- cdist/argparse.py | 11 +++--- cdist/config.py | 13 ++++--- docs/src/man1/cdist.rst | 80 +++++++++++++++++++++++------------------ 3 files changed, 60 insertions(+), 44 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index c41f4830..8e78f278 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -441,11 +441,8 @@ def get_parsers(): parser['trigger'] = parser['sub'].add_parser( 'trigger', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['config_main']]) - parser['trigger'].add_argument( - '-6', '--ipv6', default=False, - help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), - action='store_true') parser['trigger'].add_argument( '-D', '--directory', action='store', required=False, help=('Where to create local files')) @@ -453,7 +450,11 @@ def get_parsers(): '-H', '--http-port', action='store', default=3000, required=False, help=('Create trigger listener via http on specified port')) parser['trigger'].add_argument( - '-S', '--source', action='store', required=False, + '--ipv6', default=False, + help=('Listen to both IPv4 and IPv6 (instead of only IPv4)'), + action='store_true') + parser['trigger'].add_argument( + '-O', '--source', action='store', required=False, help=('Which file to copy for creation')) parser['trigger'].set_defaults(func=cdist.trigger.Trigger.commandline) diff --git a/cdist/config.py b/cdist/config.py index 26d07fc4..adf449e6 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -143,7 +143,7 @@ class Config(object): # Determine forcing IPv4/IPv6 options if any, only for # default remote commands. - if args.force_ipv: + if hasattr(args, 'force_ipv') and args.force_ipv: force_addr_opt = " -{}".format(args.force_ipv) else: force_addr_opt = "" @@ -356,10 +356,13 @@ class Config(object): @staticmethod def _address_family(args): - if args.force_ipv == 4: - family = socket.AF_INET - elif args.force_ipv == 6: - family = socket.AF_INET6 + if hasattr(args, 'force_ipv'): + if args.force_ipv == 4: + family = socket.AF_INET + elif args.force_ipv == 6: + family = socket.AF_INET6 + else: + family = 0 else: family = 0 return family diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 69b8cfc5..f213bad7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -87,12 +87,12 @@ SYNOPSIS cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] - cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] - [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] - [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] - [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] - [--remote-exec REMOTE_EXEC] [-6] [-D DIRECTORY] - [-H HTTP_PORT] [-S SOURCE] + cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] + [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] + [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] + [-S] [-D DIRECTORY] [-H HTTP_PORT] [--ipv6] [-O SOURCE] DESCRIPTION @@ -633,64 +633,76 @@ This command returns the following response codes to client requests: * 599 for cdist run errors * 500 for cdist/server errors. +**-4, -force-ipv4** + Force to use IPv4 addresses only. No influence for + custom remote commands. -**-6, --ipv6** - - Listen to both IPv4 and IPv6 (instead of only IPv4) - -**-b, --beta** - - Enable beta functionality. +**-6, --force-ipv6** + Force to use IPv6 addresses only. No influence for + custom remote commands. **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** - - Sepcify custom cache path pattern. It can also be set by - CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then - default hostdir is used. For more info on format see - :strong:`CACHE PATH PATTERN FORMAT` below. + Specify custom cache path pattern. If it is not set + then default hostdir is used. **-c CONF_DIR, --conf-dir CONF_DIR** - - Add configuration directory (can be repeated, last one wins) + Add configuration directory (can be repeated, last one + wins). **-D DIRECTORY, --directory DIRECTORY** Where to create local files -**-H HTTP_PORT, --http-port HTTP_PORT** +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. +**-H HTTP_PORT, --http-port HTTP_PORT** Create trigger listener via http on specified port **-i MANIFEST, --initial-manifest MANIFEST** + Path to a cdist manifest or '-' to read from stdin. - path to a cdist manifest or '-' to read from stdin. +**--ipv6** + Listen to both IPv4 and IPv6 (instead of only IPv4) **-j [JOBS], --jobs [JOBS]** - - Specify the maximum number of parallel jobs, currently - only global explorers are supported + Operate in parallel in specified maximum number of + jobs. Global explorers, object prepare and object run + are supported. Without argument CPU count is used by + default. Currently in beta. **-n, --dry-run** + Do not execute code. - do not execute code +**-O SOURCE, --source SOURCE** + Which file to copy for creation **-o OUT_PATH, --out-dir OUT_PATH** + Directory to save cdist output in. - directory to save cdist output in +**-P, --timestamp** + Timestamp log messages with the current local date and + time in the format: YYYYMMDDHHMMSS.us. + +**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]** + Operate by using archiving with compression where + appropriate. Supported values are: tar - tar archive, + tgz - gzip tar archive (the default), tbz2 - bzip2 tar + archive and txz - lzma tar archive. Currently in beta. **-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** - - Directory to save cdist output in on the target host + Directory to save cdist output in on the target host. **--remote-copy REMOTE_COPY** - - Command to use for remote copy (should behave like scp) + Command to use for remote copy (should behave like + scp). **--remote-exec REMOTE_EXEC** + Command to use for remote execution (should behave + like ssh). - Command to use for remote execution (should behave like ssh) +**-S, --disable-saving-output-streams** + Disable saving output streams. -**-S SOURCE, --source SOURCE** - Which file to copy for creation CONFIGURATION ------------- From 0992b1a3da04ba9db354c785c93c47eb386a57d2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Jan 2019 16:35:02 +0100 Subject: [PATCH 05/12] Add missing configuration arg --- cdist/config.py | 13 +++++-------- cdist/trigger.py | 5 ++++- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index adf449e6..26d07fc4 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -143,7 +143,7 @@ class Config(object): # Determine forcing IPv4/IPv6 options if any, only for # default remote commands. - if hasattr(args, 'force_ipv') and args.force_ipv: + if args.force_ipv: force_addr_opt = " -{}".format(args.force_ipv) else: force_addr_opt = "" @@ -356,13 +356,10 @@ class Config(object): @staticmethod def _address_family(args): - if hasattr(args, 'force_ipv'): - if args.force_ipv == 4: - family = socket.AF_INET - elif args.force_ipv == 6: - family = socket.AF_INET6 - else: - family = 0 + if args.force_ipv == 4: + family = socket.AF_INET + elif args.force_ipv == 6: + family = socket.AF_INET6 else: family = 0 return family diff --git a/cdist/trigger.py b/cdist/trigger.py index 1b9a384b..58f98dca 100644 --- a/cdist/trigger.py +++ b/cdist/trigger.py @@ -200,8 +200,11 @@ class TriggerHttp(http.server.BaseHTTPRequestHandler): log.debug("Executing cdist onehost with params: %s, %s, %s, %s, %s, ", target_host, host_tags, host_base_path, hostdir, self.cdistargs) + cfg = cdist.configuration.Configuration(self.cdistargs) + configuration = cfg.get_config(section='GLOBAL') theclass.onehost(target_host, host_tags, host_base_path, hostdir, - self.cdistargs, parallel=False) + self.cdistargs, parallel=False, + configuration=configuration) class HTTPServerV6(socketserver.ForkingMixIn, http.server.HTTPServer): From a25816e6bebbb4a9239512cf903492fce0e7babd Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 1 Feb 2019 17:49:14 +0100 Subject: [PATCH 06/12] Updated the man pages for the cdist trigger and preos --- docs/src/cdist-preos.rst | 4 ++-- docs/src/cdist-trigger.rst | 6 ++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/cdist-preos.rst b/docs/src/cdist-preos.rst index 7c0315b7..024e4bbf 100644 --- a/docs/src/cdist-preos.rst +++ b/docs/src/cdist-preos.rst @@ -26,7 +26,7 @@ For example, to create ubuntu PreOS: $ cdist preos ubuntu /preos/preos-ubuntu -B -C \ -k ~/.ssh/id_rsa.pub -p /preos/pxe-ubuntu \ - -t "/usr/bin/curl 192.168.111.5:3000/install/" + -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/" For more info about available options see cdist manual page. @@ -156,7 +156,7 @@ Tipical use case for using PreOS and trigger command include the following steps $ cdist preos ubuntu /preos/ubuntu -b -C \ -k ~/.ssh/id_rsa.pub -p /preos/pxe \ - -t "/usr/bin/curl 192.168.111.5:3000/install/" + -t "/usr/bin/curl 192.168.111.5:3000/cdist/install/" #. Configure dhcp server and tftp server. diff --git a/docs/src/cdist-trigger.rst b/docs/src/cdist-trigger.rst index 94477c29..93e9d920 100644 --- a/docs/src/cdist-trigger.rst +++ b/docs/src/cdist-trigger.rst @@ -13,8 +13,10 @@ This command starts trigger server at management node, for example: This will start cdist trigger server in verbose mode. cdist trigger server accepts simple requests for configuration and for installation: -* :strong:`/cdist/install/.*` for installation -* :strong:`/cdist/config/.*` for configuration. +* :strong:`/cdist/install/` for installation +* :strong:`/cdist/config/` for configuration. + +Other configuration parameters are the same as in like cdist config (See `cdist `_). Machines can then trigger cdist trigger server with appropriate requests. If the request is, for example, for installation (:strong:`/cdist/install/`) From 6305a7341bca15ed2a5cdfa33382624c3cc87ab0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 1 Nov 2018 17:56:40 +0100 Subject: [PATCH 07/12] Implement python types --- cdist/conf/type/__file_py/__init__.py | 103 +++++ cdist/conf/type/__file_py/explorer/cksum | 34 ++ cdist/conf/type/__file_py/explorer/stat | 56 +++ cdist/conf/type/__file_py/explorer/type | 33 ++ .../type/__file_py/parameter/default/state | 1 + cdist/conf/type/__file_py/parameter/optional | 5 + cdist/config.py | 65 ++- cdist/core/__init__.py | 1 + cdist/core/code.py | 42 ++ cdist/core/manifest.py | 75 +++- cdist/core/pytypes.py | 162 ++++++++ cdist/exec/local.py | 16 + cdist/util/python_type_util.py | 27 ++ docs/changelog | 1 + docs/dev/python-types/benchmark | 376 ++++++++++++++++++ docs/dev/python-types/benchmark.sh | 43 ++ docs/dev/python-types/conf/manifest/pyinit | 7 + docs/dev/python-types/conf/manifest/shinit | 7 + .../conf/type/__dummy_config_py/__init__.py | 22 + .../type/__dummy_config_py/files/dummy.conf | 1 + .../type/__dummy_config_sh/files/dummy.conf | 1 + .../conf/type/__dummy_config_sh/manifest | 6 + docs/dev/python-types/test.sh | 27 ++ docs/dev/python-types/timeit.sh | 36 ++ docs/src/cdist-type.rst | 86 ++++ scripts/cdist | 2 +- 26 files changed, 1224 insertions(+), 11 deletions(-) create mode 100644 cdist/conf/type/__file_py/__init__.py create mode 100755 cdist/conf/type/__file_py/explorer/cksum create mode 100755 cdist/conf/type/__file_py/explorer/stat create mode 100755 cdist/conf/type/__file_py/explorer/type create mode 100644 cdist/conf/type/__file_py/parameter/default/state create mode 100644 cdist/conf/type/__file_py/parameter/optional create mode 100644 cdist/core/pytypes.py create mode 100644 cdist/util/python_type_util.py create mode 100644 docs/dev/python-types/benchmark create mode 100755 docs/dev/python-types/benchmark.sh create mode 100644 docs/dev/python-types/conf/manifest/pyinit create mode 100644 docs/dev/python-types/conf/manifest/shinit create mode 100644 docs/dev/python-types/conf/type/__dummy_config_py/__init__.py create mode 100644 docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf create mode 100644 docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf create mode 100644 docs/dev/python-types/conf/type/__dummy_config_sh/manifest create mode 100755 docs/dev/python-types/test.sh create mode 100755 docs/dev/python-types/timeit.sh diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file_py/__init__.py new file mode 100644 index 00000000..1212f8fd --- /dev/null +++ b/cdist/conf/type/__file_py/__init__.py @@ -0,0 +1,103 @@ +import os +import re +import sys +from cdist.core.pytypes import * + + +class FileType(PythonType): + def get_attribute(self, stat_file, attribute, value_should): + if os.path.exists(stat_file): + if re.match('[0-9]', value_should): + index = 1 + else: + index = 2 + with open(stat_file, 'r') as f: + for line in f: + if re.match(attribute + ":", line): + fields = line.split() + return fields[index] + return None + + def set_attribute(self, attribute, value_should, destination): + cmd = { + 'group': 'chgrp', + 'owner': 'chown', + 'mode': 'chmod', + } + self.send_message("{} '{}'".format(cmd[attribute], value_should)) + return "{} '{}' '{}'".format(cmd[attribute], value_should, destination) + + def type_manifest(self): + yield from () + + def type_gencode(self): + typeis = self.get_explorer('type') + state_should = self.get_parameter('state') + + if state_should == 'exists' and typeis == 'file': + return + + source = self.get_parameter('source') + if source == '-': + source = self.stdin_path + destination = '/' + self.object_id + if state_should == 'pre-exists': + if source is not None: + self.die('--source cannot be used with --state pre-exists') + if typeis == 'file': + return None + else: + self.die('File {} does not exist'.format(destination)) + + create_file = False + upload_file = False + set_attributes = False + code = [] + if state_should == 'present' or state_should == 'exists': + if source is None: + remote_stat = self.get_explorer('stat') + if not remote_stat: + create_file = True + else: + if os.path.exists(source): + if typeis == 'file': + local_cksum = self.run_local(['cksum', source, ]) + local_cksum = local_cksum.split()[0] + remote_cksum = self.get_explorer('cksum') + remote_cksum = remote_cksum.split()[0] + upload_file = local_cksum != remote_cksum + else: + upload_file = True + else: + self.die('Source {} does not exist'.format(source)) + if create_file or upload_file: + set_attributes = True + tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination) + destination_upload = self.run_remote( + ["mktemp", tempfile_template, ]) + if upload_file: + self.transfer(source, destination_upload) + code.append('rm -rf {}'.format(destination)) + code.append('mv {} {}'.format(destination_upload, destination)) + + if state_should in ('present', 'exists', 'pre-exists', ): + for attribute in ('group', 'owner', 'mode', ): + if attribute in self.parameters: + value_should = self.get_parameter(attribute) + if attribute == 'mode': + value_should = re.sub('^0', '', value_should) + stat_file = self.get_explorer_file('stat') + value_is = self.get_attribute(stat_file, attribute, + value_should) + if set_attributes or value_should != value_is: + code.append(self.set_attribute(attribute, + value_should, + destination)) + elif state_should == 'absent': + if typeis == 'file': + code.append('rm -f {}'.format(destination)) + self.send_message('remove') + else: + self.die('Unknown state {}'.format(state_should)) + + return "\n".join(code) diff --git a/cdist/conf/type/__file_py/explorer/cksum b/cdist/conf/type/__file_py/explorer/cksum new file mode 100755 index 00000000..335e4e7a --- /dev/null +++ b/cdist/conf/type/__file_py/explorer/cksum @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Retrieve the md5sum of a file to be created, if it is already existing. +# + +destination="/$__object_id" + +if [ -e "$destination" ]; then + if [ -f "$destination" ]; then + cksum < "$destination" + else + echo "NO REGULAR FILE" + fi +else + echo "NO FILE FOUND, NO CHECKSUM CALCULATED." +fi diff --git a/cdist/conf/type/__file_py/explorer/stat b/cdist/conf/type/__file_py/explorer/stat new file mode 100755 index 00000000..8a917556 --- /dev/null +++ b/cdist/conf/type/__file_py/explorer/stat @@ -0,0 +1,56 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +# nothing to work with, nothing we could do +[ -e "$destination" ] || exit 0 + +os=$("$__explorer/os") +case "$os" in + "freebsd"|"netbsd"|"openbsd") + # FIXME: should be something like this based on man page, but can not test + stat -f "type: %ST +owner: %Du %Su +group: %Dg %Sg +mode: %Op %Sp +size: %Dz +links: %Dl +" "$destination" + ;; + "macosx") + stat -f "type: %HT +owner: %Du %Su +group: %Dg %Sg +mode: %Lp %Sp +size: %Dz +links: %Dl +" "$destination" + ;; + *) + stat --printf="type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; +esac diff --git a/cdist/conf/type/__file_py/explorer/type b/cdist/conf/type/__file_py/explorer/type new file mode 100755 index 00000000..e723047c --- /dev/null +++ b/cdist/conf/type/__file_py/explorer/type @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +if [ ! -e "$destination" ]; then + echo none +elif [ -h "$destination" ]; then + echo symlink +elif [ -f "$destination" ]; then + echo file +elif [ -d "$destination" ]; then + echo directory +else + echo unknown +fi diff --git a/cdist/conf/type/__file_py/parameter/default/state b/cdist/conf/type/__file_py/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__file_py/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__file_py/parameter/optional b/cdist/conf/type/__file_py/parameter/optional new file mode 100644 index 00000000..c696d592 --- /dev/null +++ b/cdist/conf/type/__file_py/parameter/optional @@ -0,0 +1,5 @@ +state +group +mode +owner +source diff --git a/cdist/config.py b/cdist/config.py index 26d07fc4..74944d16 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -38,6 +38,7 @@ import cdist.hostsource import cdist.exec.local import cdist.exec.remote import cdist.util.ipaddr as ipaddr +import cdist.util.python_type_util as pytype_util import cdist.configuration from cdist import core, inventory from cdist.util.remoteutil import inspect_ssh_mux_opts @@ -90,13 +91,15 @@ class Config(object): shutil.rmtree(path) def __init__(self, local, remote, dry_run=False, jobs=None, - cleanup_cmds=None, remove_remote_files_dirs=False): + cleanup_cmds=None, remove_remote_files_dirs=False, + timestamp=False): self.local = local self.remote = remote self._open_logger() self.dry_run = dry_run self.jobs = jobs + self.timestamp = timestamp if cleanup_cmds: self.cleanup_cmds = cleanup_cmds else: @@ -427,7 +430,8 @@ class Config(object): cleanup_cmds.append(cleanup_cmd) c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs, cleanup_cmds=cleanup_cmds, - remove_remote_files_dirs=remove_remote_files_dirs) + remove_remote_files_dirs=remove_remote_files_dirs, + timestamp=args.timestamp) c.run() cls._remove_paths() @@ -465,6 +469,7 @@ class Config(object): 'dry' if self.dry_run else 'configuration')) self._init_files_dirs() + self.local.collect_python_types() self.explorer.run_global_explorers(self.local.global_explorer_out_path) try: @@ -778,15 +783,41 @@ class Config(object): args = [param, cdist_type.name] self.log.warning(format, *args) + def _timeit(self, func, msg_prefix): + def wrapper_func(*args, **kwargs): + loglevel = self.log.getEffectiveLevel() + if loglevel >= logging.VERBOSE and self.timestamp: + start_time = time.time() + rv = func(*args, **kwargs) + end_time = time.time() + duration = end_time - start_time + self.log.verbose("%s duration: %.6f seconds", + msg_prefix, duration) + else: + rv = func(*args, **kwargs) + return rv + return wrapper_func + def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self._handle_deprecation(cdist_object) self.log.verbose("Preparing object {}".format(cdist_object.name)) self.log.verbose( "Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: - self.manifest.run_type_manifest(cdist_object) + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( + "Running manifest and explorers for " + cdist_object.name) + self.explorer.run_type_explorers(cdist_object, + transfer_type_explorers) + if pytype_util.is_python_type(cdist_object.cdist_type): + self._timeit(self.manifest.run_py_type_manifest, + "Python type manifest for {}".format( + cdist_object.name))(cdist_object) + else: + self._timeit(self.manifest.run_type_manifest, + "Type manifest for {}".format( + cdist_object.name))(cdist_object) cdist_object.state = core.CdistObject.STATE_PREPARED except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) @@ -801,9 +832,21 @@ class Config(object): # Generate self.log.debug("Generating code for %s" % (cdist_object.name)) - cdist_object.code_local = self.code.run_gencode_local(cdist_object) - cdist_object.code_remote = self.code.run_gencode_remote( - cdist_object) + if pytype_util.is_python_type(cdist_object.cdist_type): + cdist_object.code_local = '' + cdist_object.code_remote = self._timeit( + self.code.run_py, + "Python type generate code for {}".format( + cdist_object.name))(cdist_object) + else: + cdist_object.code_local = self._timeit( + self.code.run_gencode_local, + "Type generate code local for {}".format( + cdist_object.name))(cdist_object) + cdist_object.code_remote = self._timeit( + self.code.run_gencode_remote, + "Type generate code remote for {}".format( + cdist_object.name))(cdist_object) if cdist_object.code_local or cdist_object.code_remote: cdist_object.changed = True @@ -814,12 +857,16 @@ class Config(object): if cdist_object.code_local: self.log.trace("Executing local code for %s" % (cdist_object.name)) - self.code.run_code_local(cdist_object) + self._timeit(self.code.run_code_local, + "Type run code local for {}".format( + cdist_object.name))(cdist_object) if cdist_object.code_remote: self.log.trace("Executing remote code for %s" % (cdist_object.name)) self.code.transfer_code_remote(cdist_object) - self.code.run_code_remote(cdist_object) + self._timeit(self.code.run_code_remote, + "Type run code remote for {}".format( + cdist_object.name))(cdist_object) # Mark this object as done self.log.trace("Finishing run of " + cdist_object.name) diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index b79cdb21..de2fbbbf 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -29,3 +29,4 @@ from cdist.core.manifest import Manifest from cdist.core.code import Code from cdist.core.util import listdir from cdist.core.util import log_level_env_var_val, log_level_name_env_var_val +import cdist.core.pytypes diff --git a/cdist/core/code.py b/cdist/core/code.py index 1550880a..34d5bd97 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -22,6 +22,10 @@ # import os +import importlib.util +import inspect +import cdist +from cdist.core.pytypes import PythonType from . import util @@ -116,6 +120,44 @@ class Code(object): if dry_run: self.env['__cdist_dry_run'] = '1' + def run_py(self, cdist_object): + cdist_type = cdist_object.cdist_type + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + + if os.path.isfile(file_path): + spec = importlib.util.spec_from_file_location(module_name, + file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + type_class = None + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error("Only one python type class is " + "supported, but at least two " + "found: {}".format((type_class, + cl, ))) + else: + type_class = cl + env = os.environ.copy() + env.update(self.env) + message_prefix = cdist_object.name + type_obj = type_class(env=env, cdist_object=cdist_object, + local=self.local, remote=self.remote, + message_prefix=message_prefix) + if hasattr(type_obj, 'run') and inspect.ismethod(type_obj.run): + if self.local.save_output_streams: + which = 'gencode-py' + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'a+') as stderr, \ + open(stdout_path, 'a+') as stdout: + return type_obj.run(stdout=stdout, stderr=stderr) + else: + return type_obj.run() + def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type script = os.path.join(self.local.type_path, diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 07af0ef8..279f5cf9 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -22,9 +22,13 @@ import logging import os - +import importlib.util +import inspect import cdist +import cdist.emulator from . import util +from cdist.core.pytypes import PythonType, Command + ''' common: @@ -212,3 +216,72 @@ class Manifest(object): type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix) + + def env_py_type_manifest(self, cdist_object): + env = os.environ.copy() + env.update(self.env) + env.update({ + '__cdist_object_marker': self.local.object_marker_name, + '__cdist_manifest': cdist_object.cdist_type, + '__manifest': self.local.manifest_path, + '__object': cdist_object.absolute_path, + '__object_id': cdist_object.object_id, + '__object_name': cdist_object.name, + '__type': cdist_object.cdist_type.absolute_path, + }) + + return env + + def run_py_type_manifest(self, cdist_object): + cdist_type = cdist_object.cdist_type + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + message_prefix = cdist_object.name + if os.path.isfile(file_path): + self.log.verbose("Running python type manifest for object %s", + cdist_object.name) + spec = importlib.util.spec_from_file_location(module_name, + file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + type_class = None + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error("Only one python type class is " + "supported, but at least two " + "found: {}".format((type_class, + cl, ))) + else: + type_class = cl + env = self.env_py_type_manifest(cdist_object) + type_obj = type_class(env=env, cdist_object=cdist_object, + local=self.local, remote=None, + message_prefix=message_prefix) + if self.local.save_output_streams: + which = 'manifest' + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'a+') as stderr, \ + open(stdout_path, 'a+') as stdout: + self._process_py_type_manifest_entries( + type_obj, env, stdout=stdout, stderr=stderr) + else: + self._process_py_type_manifest_entries(type_obj, env) + + def _process_py_type_manifest_entries(self, type_obj, env, stdout=None, + stderr=None): + if hasattr(type_obj, 'manifest') and \ + inspect.ismethod(type_obj.manifest): + for cmd in type_obj.manifest(stdout=stdout, stderr=stderr): + if not isinstance(cmd, Command): + raise TypeError("Manifest command must be of type Command") + kwargs = { + 'argv': cmd.cmd_line(), + 'env': env, + } + if cmd.stdin: + kwargs['stdin'] = cmd.stdin + emulator = cdist.emulator.Emulator(**kwargs) + emulator.run() diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py new file mode 100644 index 00000000..ae4164ec --- /dev/null +++ b/cdist/core/pytypes.py @@ -0,0 +1,162 @@ +import logging +import os +import io +import sys +import re +from cdist import message, Error + + +__all__ = ["PythonType", "Command", "command"] + + +class PythonType: + def __init__(self, env, cdist_object, local, remote, message_prefix=None): + self.env = env + self.cdist_object = cdist_object + self.object_id = cdist_object.object_id + self.object_name = cdist_object.name + self.cdist_type = cdist_object.cdist_type + self.local = local + self.remote = remote + self.object_path = cdist_object.absolute_path + self.type_path = cdist_object.cdist_type.absolute_path + self.explorer_path = os.path.join(self.object_path, 'explorer') + self.parameters = cdist_object.parameters + self.stdin_path = os.path.join(self.object_path, 'stdin') + self.log = logging.getLogger( + self.local.target_host[0] + ':' + self.object_name) + + self.message_prefix = message_prefix + self.message = None + + def get_parameter(self, name): + return self.parameters.get(name) + + def get_explorer_file(self, name): + path = os.path.join(self.explorer_path, name) + return path + + def get_explorer(self, name): + path = self.get_explorer_file(name) + with open(path, 'r') as f: + value = f.read() + if value: + value = value.strip() + return value + + def run_local(self, command, env=None): + rv = self.local.run(command, env=env, return_output=True) + if rv: + rv = rv.rstrip('\n') + return rv + + def run_remote(self, command, env=None): + rv = self.remote.run(command, env=env, return_output=True) + if rv: + rv = rv.rstrip('\n') + return rv + + def transfer(self, source, destination): + self.remote.transfer(source, destination) + + def die(self, msg): + raise Error("{}: {}".format(self.cdist_object, msg)) + + def type_manifest(self): + pass + + def type_gencode(self): + pass + + def manifest(self, stdout=None, stderr=None): + try: + if self.message_prefix: + self.message = message.Message(self.message_prefix, + self.local.messages_path) + self.env.update(self.message.env) + if stdout is not None: + stdout_save = sys.stdout + sys.stdout = stdout + if stderr is not None: + stderr_save = sys.stderr + sys.stderr = stderr + yield from self.type_manifest() + finally: + if self.message: + self.message.merge_messages() + if stdout is not None: + sys.stdout = stdout_save + if stderr is not None: + sys.stderr = stderr_save + + def run(self, stdout=None, stderr=None): + try: + if self.message_prefix: + self.message = message.Message(self.message_prefix, + self.local.messages_path) + if stdout is not None: + stdout_save = sys.stdout + sys.stdout = stdout + if stderr is not None: + stderr_save = sys.stderr + sys.stderr = stderr + return self.type_gencode() + finally: + if self.message: + self.message.merge_messages() + if stdout is not None: + sys.stdout = stdout_save + if stderr is not None: + sys.stderr = stderr_save + + def send_message(self, msg): + if self.message: + with open(self.message.messages_out, 'a') as f: + print(msg, file=f) + + def receive_message(self, pattern): + if self.message: + with open(self.message.messages_in, 'r') as f: + for line in f: + match = re.search(pattern, line) + if match: + return match + return None + + +class Command: + def __init__(self, name, *args, **kwargs): + self.name = name + self.args = args + self.kwargs = kwargs + self.stdin = None + + def feed_stdin(self, value): + # If file-like object then read its value. + if value is not None and isinstance(value, io.IOBase): + value = value.read() + + # Convert to bytes file-like object. + if value is None: + self.stdin = None + elif isinstance(value, str): + self.stdin = io.BytesIO(value.encode('utf-8')) + elif isinstance(value, bytes) or isinstance(value, bytearray): + self.stdin = io.BytesIO(value) + else: + raise TypeError("value must be str, bytes, bytearray, file-like " + "object or None") + return self + + def cmd_line(self): + argv = [self.name, ] + for param in self.args: + argv.append(param) + for key, value in self.kwargs.items(): + argv.append("--{}".format(key)) + argv.append(value) + return argv + + +def command(name, *args, **kwargs): + return Command(name, *args, **kwargs) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index f83c85df..2cc9d15b 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -35,6 +35,9 @@ import cdist import cdist.message from cdist import core import cdist.exec.util as util +import cdist.util.python_type_util as pytype_util +from cdist.core import pytypes +import functools CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] @@ -398,3 +401,16 @@ class Local(object): raise cdist.Error( "Linking emulator from %s to %s failed: %s" % ( src, dst, e.__str__())) + + def collect_python_types(self): + for cdist_type in core.CdistType.list_types(self.type_path): + if pytype_util.is_python_type(cdist_type): + self.log.trace("Detected python type %s, collecting it".format( + cdist_type.name)) + f = functools.partial(pytypes.command, cdist_type.name) + if cdist_type.name.startswith('__'): + attr_name = cdist_type.name.replace('__', '', 1) + else: + attr_name = cdist_type.name + setattr(pytypes, attr_name, f) + pytypes.__all__.append(attr_name) diff --git a/cdist/util/python_type_util.py b/cdist/util/python_type_util.py new file mode 100644 index 00000000..fd97684e --- /dev/null +++ b/cdist/util/python_type_util.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# 2019 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + + +def is_python_type(cdist_type): + init_path = os.path.join(cdist_type.absolute_path, '__init__.py') + return os.path.exists(init_path) diff --git a/docs/changelog b/docs/changelog index 41213464..b8bdd899 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Core: Add trigger functionality (Nico Schottelius, Darko Poljak) + * Core: Implement core support for python types (Darko Poljak) 6.1.0: 2019-11-19 * Explorer hostname, type __hostname: Support more operating systems, rewrite type and hostname explorer (Dennis Camera) diff --git a/docs/dev/python-types/benchmark b/docs/dev/python-types/benchmark new file mode 100644 index 00000000..8fcf69f3 --- /dev/null +++ b/docs/dev/python-types/benchmark @@ -0,0 +1,376 @@ +# sh type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125154.045799] 185.203.112.26: Starting configuration run +INFO: [20181121125237.029892] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121125239.881529] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121125243.265672] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121125246.929903] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121125251.811258] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121125257.784416] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121125302.686275] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121125306.394030] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121125308.610023] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121125310.868538] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121125313.017972] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121125315.201342] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121125317.333055] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121125319.463929] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121125321.595410] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121125323.689697] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121125325.768283] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121125327.814793] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121125329.873073] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121125331.953886] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121125334.118290] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121125336.390849] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121125338.576698] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121125340.819044] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121125343.680419] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121125346.044907] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121125348.179574] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121125350.314970] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121125352.447394] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121125354.586637] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121125356.722699] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121125358.883538] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121125401.020967] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121125403.160146] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121125405.289048] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121125407.423994] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121125409.530135] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121125411.659683] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121125413.786177] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121125415.919152] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121125418.051496] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121125420.204577] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121125422.339697] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121125424.450966] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121125426.487831] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121125428.585516] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121125430.749002] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121125432.865290] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121125435.004009] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121125437.228566] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds + +# sh type, files exist at remote but content changes +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125529.952800] 185.203.112.26: Starting configuration run +INFO: [20181121125541.175180] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121125543.219561] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121125545.116073] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121125547.011359] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121125548.916288] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121125550.821351] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121125552.723887] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121125554.635662] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121125556.568639] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121125558.508852] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121125600.464475] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121125602.429261] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121125604.428942] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121125606.442193] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121125608.474473] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121125610.535252] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121125612.609560] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121125614.708507] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121125616.824721] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121125618.924521] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121125621.007543] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121125623.133204] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121125625.333471] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121125627.396334] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121125629.526492] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121125631.628454] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121125633.743142] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121125635.952547] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121125637.986746] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121125640.020415] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121125642.081373] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121125644.174744] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121125646.286532] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121125648.396447] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121125650.460107] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121125652.557125] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121125654.667456] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121125656.746960] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121125658.854229] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121125700.968145] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121125703.109376] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121125705.318163] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121125707.440575] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121125709.551261] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121125711.657753] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121125713.774819] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121125715.887428] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121125717.995104] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121125720.110196] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121125722.232932] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds + +# py type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121125812.034197] 185.203.112.26: Starting configuration run +INFO: [20181121125823.927353] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121125825.715361] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121125827.511296] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121125829.293455] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121125831.086696] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121125832.867582] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121125834.652511] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121125836.450393] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121125838.255152] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121125840.065808] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121125841.889049] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121125843.719280] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121125845.560165] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121125847.416138] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121125849.289851] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121125851.180203] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121125853.074978] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121125855.086107] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121125857.041100] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121125859.025581] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121125901.072067] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121125903.026711] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121125904.994824] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121125906.956296] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121125908.929231] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121125910.882672] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121125912.839834] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121125914.789904] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121125916.743930] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121125918.698258] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121125920.657118] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121125922.618898] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121125924.567847] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121125926.524617] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121125928.396400] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121125930.209237] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121125931.998377] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121125933.786883] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121125935.579348] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121125937.366197] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121125939.155643] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121125941.052837] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121125942.953670] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121125944.781567] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121125946.622485] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121125948.470701] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121125950.356949] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121125952.232014] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121125954.128887] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121125956.037541] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds + +# py type, files exist at remote but content changes +echo 'x=0; while [ $x -lt 50 ]; do head -c 102400 /dev/random | __file_py /root/foo${x}.bin --source - --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130056.484643] 185.203.112.26: Starting configuration run +INFO: [20181121130108.545059] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121130110.339217] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121130112.136448] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121130113.923820] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121130115.715667] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121130117.508696] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121130119.300839] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121130124.296312] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121130131.109195] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121130133.303817] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121130136.396440] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121130138.443128] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121130140.462868] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121130142.476196] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121130145.937900] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121130148.013672] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121130150.042588] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121130152.050793] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121130154.083089] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121130156.100091] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121130158.103005] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121130200.188390] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121130202.197574] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121130205.269102] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121130208.457011] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121130211.574321] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121130213.719894] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121130215.762977] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121130217.778624] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121130219.840477] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121130221.852389] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121130223.850898] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121130225.858812] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121130227.855295] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121130229.952673] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121130231.956904] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121130233.961954] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121130236.012158] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121130238.024422] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121130241.238800] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121130243.463237] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121130245.610314] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121130247.661385] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121130250.399845] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121130252.832133] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121130254.955658] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121130257.039587] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121130259.178847] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121130301.357922] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121130303.356299] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds + + + +# init test file content +head -c 102400 /dev/random > /tmp/test.file + +# sh type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130612.519698] 185.203.112.26: Starting configuration run +INFO: [20181121130624.219344] 185.203.112.26: Processing __file/root/foo0.bin +INFO: [20181121130626.980052] 185.203.112.26: Processing __file/root/foo1.bin +INFO: [20181121130631.200669] 185.203.112.26: Processing __file/root/foo2.bin +INFO: [20181121130642.790229] 185.203.112.26: Processing __file/root/foo3.bin +INFO: [20181121130646.565599] 185.203.112.26: Processing __file/root/foo4.bin +INFO: [20181121130648.724875] 185.203.112.26: Processing __file/root/foo5.bin +INFO: [20181121130651.464686] 185.203.112.26: Processing __file/root/foo6.bin +INFO: [20181121130653.639581] 185.203.112.26: Processing __file/root/foo7.bin +INFO: [20181121130655.773987] 185.203.112.26: Processing __file/root/foo8.bin +INFO: [20181121130657.933136] 185.203.112.26: Processing __file/root/foo9.bin +INFO: [20181121130700.065158] 185.203.112.26: Processing __file/root/foo10.bin +INFO: [20181121130702.216456] 185.203.112.26: Processing __file/root/foo11.bin +INFO: [20181121130704.429030] 185.203.112.26: Processing __file/root/foo12.bin +INFO: [20181121130706.562114] 185.203.112.26: Processing __file/root/foo13.bin +INFO: [20181121130708.696584] 185.203.112.26: Processing __file/root/foo14.bin +INFO: [20181121130710.830002] 185.203.112.26: Processing __file/root/foo15.bin +INFO: [20181121130712.966631] 185.203.112.26: Processing __file/root/foo16.bin +INFO: [20181121130715.151833] 185.203.112.26: Processing __file/root/foo17.bin +INFO: [20181121130717.355196] 185.203.112.26: Processing __file/root/foo18.bin +INFO: [20181121130719.486316] 185.203.112.26: Processing __file/root/foo19.bin +INFO: [20181121130721.619933] 185.203.112.26: Processing __file/root/foo20.bin +INFO: [20181121130723.786670] 185.203.112.26: Processing __file/root/foo21.bin +INFO: [20181121130725.924736] 185.203.112.26: Processing __file/root/foo22.bin +INFO: [20181121130728.060224] 185.203.112.26: Processing __file/root/foo23.bin +INFO: [20181121130730.178729] 185.203.112.26: Processing __file/root/foo24.bin +INFO: [20181121130732.309264] 185.203.112.26: Processing __file/root/foo25.bin +INFO: [20181121130734.479895] 185.203.112.26: Processing __file/root/foo26.bin +INFO: [20181121130736.653085] 185.203.112.26: Processing __file/root/foo27.bin +INFO: [20181121130738.814291] 185.203.112.26: Processing __file/root/foo28.bin +INFO: [20181121130741.029646] 185.203.112.26: Processing __file/root/foo29.bin +INFO: [20181121130743.128717] 185.203.112.26: Processing __file/root/foo30.bin +INFO: [20181121130745.233272] 185.203.112.26: Processing __file/root/foo31.bin +INFO: [20181121130747.364681] 185.203.112.26: Processing __file/root/foo32.bin +INFO: [20181121130749.491793] 185.203.112.26: Processing __file/root/foo33.bin +INFO: [20181121130751.620492] 185.203.112.26: Processing __file/root/foo34.bin +INFO: [20181121130753.743519] 185.203.112.26: Processing __file/root/foo35.bin +INFO: [20181121130755.862169] 185.203.112.26: Processing __file/root/foo36.bin +INFO: [20181121130758.000172] 185.203.112.26: Processing __file/root/foo37.bin +INFO: [20181121130800.090405] 185.203.112.26: Processing __file/root/foo38.bin +INFO: [20181121130802.211849] 185.203.112.26: Processing __file/root/foo39.bin +INFO: [20181121130804.356363] 185.203.112.26: Processing __file/root/foo40.bin +INFO: [20181121130806.548412] 185.203.112.26: Processing __file/root/foo41.bin +INFO: [20181121130808.671279] 185.203.112.26: Processing __file/root/foo42.bin +INFO: [20181121130810.752813] 185.203.112.26: Processing __file/root/foo43.bin +INFO: [20181121130812.844502] 185.203.112.26: Processing __file/root/foo44.bin +INFO: [20181121130814.950501] 185.203.112.26: Processing __file/root/foo45.bin +INFO: [20181121130817.040587] 185.203.112.26: Processing __file/root/foo46.bin +INFO: [20181121130819.175850] 185.203.112.26: Processing __file/root/foo47.bin +INFO: [20181121130821.332900] 185.203.112.26: Processing __file/root/foo48.bin +INFO: [20181121130823.543119] 185.203.112.26: Processing __file/root/foo49.bin +INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds + +# sh type, files exist at remote +echo 'x=0; while [ $x -lt 50 ]; do __file /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121130854.980007] 185.203.112.26: Starting configuration run +INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds + +# py type, no file at remote +echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121131110.179480] 185.203.112.26: Starting configuration run +INFO: [20181121131122.086849] 185.203.112.26: Processing __file_py/root/foo0.bin +INFO: [20181121131123.876029] 185.203.112.26: Processing __file_py/root/foo1.bin +INFO: [20181121131125.668664] 185.203.112.26: Processing __file_py/root/foo2.bin +INFO: [20181121131127.460721] 185.203.112.26: Processing __file_py/root/foo3.bin +INFO: [20181121131129.591229] 185.203.112.26: Processing __file_py/root/foo4.bin +INFO: [20181121131131.390379] 185.203.112.26: Processing __file_py/root/foo5.bin +INFO: [20181121131133.195275] 185.203.112.26: Processing __file_py/root/foo6.bin +INFO: [20181121131135.006282] 185.203.112.26: Processing __file_py/root/foo7.bin +INFO: [20181121131136.834448] 185.203.112.26: Processing __file_py/root/foo8.bin +INFO: [20181121131138.659301] 185.203.112.26: Processing __file_py/root/foo9.bin +INFO: [20181121131140.496856] 185.203.112.26: Processing __file_py/root/foo10.bin +INFO: [20181121131142.367813] 185.203.112.26: Processing __file_py/root/foo11.bin +INFO: [20181121131144.239817] 185.203.112.26: Processing __file_py/root/foo12.bin +INFO: [20181121131146.133314] 185.203.112.26: Processing __file_py/root/foo13.bin +INFO: [20181121131148.049380] 185.203.112.26: Processing __file_py/root/foo14.bin +INFO: [20181121131149.974696] 185.203.112.26: Processing __file_py/root/foo15.bin +INFO: [20181121131151.929083] 185.203.112.26: Processing __file_py/root/foo16.bin +INFO: [20181121131153.923590] 185.203.112.26: Processing __file_py/root/foo17.bin +INFO: [20181121131155.874910] 185.203.112.26: Processing __file_py/root/foo18.bin +INFO: [20181121131157.857904] 185.203.112.26: Processing __file_py/root/foo19.bin +INFO: [20181121131159.902006] 185.203.112.26: Processing __file_py/root/foo20.bin +INFO: [20181121131201.859840] 185.203.112.26: Processing __file_py/root/foo21.bin +INFO: [20181121131203.810875] 185.203.112.26: Processing __file_py/root/foo22.bin +INFO: [20181121131205.763291] 185.203.112.26: Processing __file_py/root/foo23.bin +INFO: [20181121131207.710932] 185.203.112.26: Processing __file_py/root/foo24.bin +INFO: [20181121131209.658154] 185.203.112.26: Processing __file_py/root/foo25.bin +INFO: [20181121131211.615374] 185.203.112.26: Processing __file_py/root/foo26.bin +INFO: [20181121131213.569721] 185.203.112.26: Processing __file_py/root/foo27.bin +INFO: [20181121131215.522624] 185.203.112.26: Processing __file_py/root/foo28.bin +INFO: [20181121131217.471128] 185.203.112.26: Processing __file_py/root/foo29.bin +INFO: [20181121131219.421712] 185.203.112.26: Processing __file_py/root/foo30.bin +INFO: [20181121131221.375699] 185.203.112.26: Processing __file_py/root/foo31.bin +INFO: [20181121131223.327672] 185.203.112.26: Processing __file_py/root/foo32.bin +INFO: [20181121131225.281373] 185.203.112.26: Processing __file_py/root/foo33.bin +INFO: [20181121131227.256711] 185.203.112.26: Processing __file_py/root/foo34.bin +INFO: [20181121131229.209255] 185.203.112.26: Processing __file_py/root/foo35.bin +INFO: [20181121131231.170170] 185.203.112.26: Processing __file_py/root/foo36.bin +INFO: [20181121131233.123407] 185.203.112.26: Processing __file_py/root/foo37.bin +INFO: [20181121131235.077713] 185.203.112.26: Processing __file_py/root/foo38.bin +INFO: [20181121131237.017138] 185.203.112.26: Processing __file_py/root/foo39.bin +INFO: [20181121131238.988189] 185.203.112.26: Processing __file_py/root/foo40.bin +INFO: [20181121131241.026849] 185.203.112.26: Processing __file_py/root/foo41.bin +INFO: [20181121131242.978335] 185.203.112.26: Processing __file_py/root/foo42.bin +INFO: [20181121131244.934562] 185.203.112.26: Processing __file_py/root/foo43.bin +INFO: [20181121131246.885320] 185.203.112.26: Processing __file_py/root/foo44.bin +INFO: [20181121131248.835008] 185.203.112.26: Processing __file_py/root/foo45.bin +INFO: [20181121131250.789727] 185.203.112.26: Processing __file_py/root/foo46.bin +INFO: [20181121131252.738686] 185.203.112.26: Processing __file_py/root/foo47.bin +INFO: [20181121131254.691465] 185.203.112.26: Processing __file_py/root/foo48.bin +INFO: [20181121131256.640896] 185.203.112.26: Processing __file_py/root/foo49.bin +INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds + +# py type, files exist at remote +echo 'x=0; while [ $x -lt 50 ]; do __file_py /root/foo${x}.bin --source /tmp/test.file --mode 0640 --owner root --group root; x=$((x + 1)); done' | ./bin/cdist config -v -P -i - 185.203.112.26 + +INFO: [20181121131327.054523] 185.203.112.26: Starting configuration run +INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds + + +# Summary + +# sh type, no file at remote +INFO: [20181121125439.429440] 185.203.112.26: Finished successful run in 165.38 seconds +# py type, no file at remote +INFO: [20181121125957.514738] 185.203.112.26: Finished successful run in 105.48 seconds + +# sh type, files exist at remote but content changes +INFO: [20181121125724.451523] 185.203.112.26: Finished successful run in 114.50 seconds +# py type, files exist at remote but content changes +INFO: [20181121130305.144393] 185.203.112.26: Finished successful run in 128.66 seconds + + +# sh type, no file at remote +INFO: [20181121130825.833163] 185.203.112.26: Finished successful run in 133.31 seconds +# py type, no file at remote +INFO: [20181121131258.194372] 185.203.112.26: Finished successful run in 108.01 seconds + +# sh type, files exist at remote +INFO: [20181121130957.927705] 185.203.112.26: Finished successful run in 62.95 seconds +# py type, files exist at remote +INFO: [20181121131428.031761] 185.203.112.26: Finished successful run in 60.98 seconds diff --git a/docs/dev/python-types/benchmark.sh b/docs/dev/python-types/benchmark.sh new file mode 100755 index 00000000..3e01941f --- /dev/null +++ b/docs/dev/python-types/benchmark.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 + +if [ $# -eq 0 ] +then + N=1 +else + N=$1 +fi + +manifest() { + bytes=$(echo "$1 * 1024" | bc) + echo "head -c ${bytes} /dev/random | __file$2 /root/foo$3.bin --source - --mode 0640 --owner root --group root" +} + +verbosity="-vv" #"-vvv" +i=0 +while [ "$i" -lt "$N" ] +do + if [ "$N" -ne 1 ] + then + printf "iteration %d\\n" "$i" + fi + printf "shinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foo$i.bin;' + manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "pyinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foo$i.bin;' + manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "shinit present state...\\n" + manifest 50 "" $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + printf "pyinit present state...\\n" + manifest 50 '_py' $i | ./bin/cdist config "${verbosity}" -P -i - ${TARGET_HOST} + + i=$((i + 1)) +done diff --git a/docs/dev/python-types/conf/manifest/pyinit b/docs/dev/python-types/conf/manifest/pyinit new file mode 100644 index 00000000..53f15a97 --- /dev/null +++ b/docs/dev/python-types/conf/manifest/pyinit @@ -0,0 +1,7 @@ +#for x in 1; do +# echo xxx${x} | __file_py /root/foobar${x} --source - --mode 0640 --owner root --group root; +#done +#__dummy_config_py test1 + +echo xxx | __file_py /root/foobar --source - --mode 0640 --owner root --group root +__dummy_config_py test1 diff --git a/docs/dev/python-types/conf/manifest/shinit b/docs/dev/python-types/conf/manifest/shinit new file mode 100644 index 00000000..44129546 --- /dev/null +++ b/docs/dev/python-types/conf/manifest/shinit @@ -0,0 +1,7 @@ +#for x in 1; do +# echo xxx${x} | __file /root/foobar${x} --source - --mode 0640 --owner root --group root; +#done +#__dummy_config_sh test1 + +echo xxx | __file /root/foobar --source - --mode 0640 --owner root --group root +__dummy_config_sh test1 diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py new file mode 100644 index 00000000..0b5fa692 --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_py/__init__.py @@ -0,0 +1,22 @@ +import os +import sys +from cdist.core.pytypes import * + + +class DummyConfig(PythonType): + def type_manifest(self): + print('dummy manifest stdout') + print('dummy manifest stderr\n', file=sys.stderr) + yield file_py('/root/dummy1.conf', + mode='0640', + owner='root', + group='root', + source='-').feed_stdin('dummy=1\n') + + self_path = os.path.dirname(os.path.realpath(__file__)) + conf_path = os.path.join(self_path, 'files', 'dummy.conf') + yield file_py('/root/dummy2.conf', + mode='0640', + owner='root', + group='root', + source=conf_path) diff --git a/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf new file mode 100644 index 00000000..972d11ca --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_py/files/dummy.conf @@ -0,0 +1 @@ +dummy=2 diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf new file mode 100644 index 00000000..972d11ca --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_sh/files/dummy.conf @@ -0,0 +1 @@ +dummy=2 diff --git a/docs/dev/python-types/conf/type/__dummy_config_sh/manifest b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest new file mode 100644 index 00000000..d675d6a3 --- /dev/null +++ b/docs/dev/python-types/conf/type/__dummy_config_sh/manifest @@ -0,0 +1,6 @@ +printf 'dummy manifest stdout\n' +printf 'dummy manifest stderr\n' >&2 + +printf "dummy=1\\n" | __file /root/dummy1.conf --mode 0640 --owner root --group root --source - + +__file /root/dummy2.conf --mode 0600 --owner root --group root --source "$__type/files/dummy.conf" diff --git a/docs/dev/python-types/test.sh b/docs/dev/python-types/test.sh new file mode 100755 index 00000000..941ea264 --- /dev/null +++ b/docs/dev/python-types/test.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 +env | grep CDIST_PATH + +for streams in ' ' '-S' +do + for x in sh py + do + printf "[%s] Removing old foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*' + printf "[%s] Listing foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'ls foobar* dummy*' + printf "[%s] Running cdist config, streams: %s\\n" "$x" "$streams" + printf -- "----------------\\n" + ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST} + printf "[%s] Listing foobar* files\\n" "$x" + printf -- "----------------\\n" + ssh root@${TARGET_HOST} 'ls foobar* dummy*' + ./bin/cdist config -P ${streams} -v -i ./docs/dev/python-types/conf/manifest/${x}init -- ${TARGET_HOST} + done +done diff --git a/docs/dev/python-types/timeit.sh b/docs/dev/python-types/timeit.sh new file mode 100755 index 00000000..f1b6c9fb --- /dev/null +++ b/docs/dev/python-types/timeit.sh @@ -0,0 +1,36 @@ +#!/bin/sh + +# Addapt to your env. +CDIST_PATH="$CDIST_PATH:./docs/dev/python-types/conf" +export CDIST_PATH +TARGET_HOST=185.203.112.26 + +if [ $# -eq 0 ] +then + N=1 +else + N=$1 +fi + +i=0 +while [ "$i" -lt "$N" ] +do + if [ "$N" -ne 1 ] + then + printf "iteration %d\\n" "$i" + fi + printf "shinit clean state...\\n" + ssh root@${TARGET_HOST} 'rm foobar*; rm dummy*;' + + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST} + printf "pyinit clean state...\\n" + ssh root@$${TARGET_HOST} 'rm foobar*; rm dummy*;' + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST} + + printf "shinit present state...\\n" + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/shinit ${TARGET_HOST} + + printf "pyinit present state...\\n" + time ./bin/cdist config -vv -P -i ./docs/dev/python-types/conf/manifest/pyinit ${TARGET_HOST} + i=$((i + 1)) +done diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 582c0938..3a4a0b13 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -522,3 +522,89 @@ How to include a type into upstream cdist If you think your type may be useful for others, ensure it works with the current master branch of cdist and have a look at `cdist hacking `_ on how to submit it. + + +Python types +------------ +From version/branch **beta** cdist support python types, types that are written +in python language with cdist's core support. cdist detects such type if type is +detectable as a python package, i.e. if **__init__.py** file is present in type's +root directory. Upon that detection cdist will try to run such type as core python +type. + +Note that this differs from plain cdist type where scripts are written in pure +python and have a proper shebang. + +Core python types replace manifest and gencode scripts. Parameters, singleton, +nonparallel are still defined as for common types. Explorer code is also written +in shell, since this is the code that is directly executed at target host. + +When writing python type you can extend **cdist.core.pytypes.PythonType** class. +You need to implement the following methods: + +* **type_manifest**: implementation should yield **cdist.core.pytypes.** + attribute function call result, or **yield from ()** if type does not use other types +* **type_gencode**: implementation should return a string consisting of lines + of shell code that will be executed at target host. + +**cdist.core.pytypes.** attributes correspond to detected python types. +**Note** that double underscore ('__') at the beginning of type name is removed. + +Example: + +.. code-block:: sh + + import os + import sys + from cdist.core.pytypes import * + + + class DummyConfig(PythonType): + def type_manifest(self): + print('dummy manifest stdout') + print('dummy manifest stderr\n', file=sys.stderr) + yield file_py('/root/dummy1.conf', + mode='0640', + owner='root', + group='root', + source='-').feed_stdin('dummy=1\n') + + self_path = os.path.dirname(os.path.realpath(__file__)) + conf_path = os.path.join(self_path, 'files', 'dummy.conf') + yield file_py('/root/dummy2.conf', + mode='0640', + owner='root', + group='root', + source=conf_path) + +**cdist.core.PythonType** class provides the following methods: + +* **get_parameter**: get type parameter +* **get_explorer_file**: get path to file for specified explorer +* **get_explorer**: get value for specified explorer +* **run_local**: run specified command locally +* **run_remote**: run specified command remotely +* **transfer**: transfer specified source to the remote +* **die**: raise error +* **send_message**: send message +* **receive_message**: get message. + +When running python type, cdist will save output streams to **gencode-py**, +stdout and stderr output files. + +As a reference implementation you can take a look at **__file_py** type, +which is re-implementation of **__file** type. + +Furthermore, under **docs/dev/python-types** there are sample cdist conf directory, +init manifests and scripts for running and measuring duration of samples. +There, under **conf/type/__dummy_config** you can find another example of +python type, which (unlike **__file_py** type) also uses new manifest implementation +that yields **cdist.core.pytypes.** attribute function call results. + +**NOTE** that python types implementation is under the beta, not directly controled by +the **-b/--beta** option. It is controled by the explicit usage of python types in +your config. + +Also, this documenation is only an introduction, and not a complete guide to python +types. Currently, it is just a short introduction so one can start to write and use +python types. diff --git a/scripts/cdist b/scripts/cdist index 7bf12c01..664504a0 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -60,7 +60,7 @@ def commandline(): if __name__ == "__main__": - cdistpythonversion = '3.2' + cdistpythonversion = '3.5' if sys.version < cdistpythonversion: print('Python >= {} is required on the source host.'.format( cdistpythonversion), file=sys.stderr) From 94ca3adcf6c2c1772a2c535e369f3c1d9d0371c4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 22 Jun 2019 12:55:56 +0200 Subject: [PATCH 08/12] Add support for python type defined argument parser --- cdist/conf/type/__file_py/__init__.py | 14 ++++++- cdist/core/code.py | 25 ++--------- cdist/core/manifest.py | 25 ++--------- cdist/core/pytypes.py | 60 ++++++++++++++++++++------- cdist/emulator.py | 34 ++++++++++++--- docs/src/cdist-type.rst | 3 ++ 6 files changed, 96 insertions(+), 65 deletions(-) diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file_py/__init__.py index 1212f8fd..3397b417 100644 --- a/cdist/conf/type/__file_py/__init__.py +++ b/cdist/conf/type/__file_py/__init__.py @@ -1,7 +1,7 @@ import os import re -import sys from cdist.core.pytypes import * +import argparse class FileType(PythonType): @@ -101,3 +101,15 @@ class FileType(PythonType): self.die('Unknown state {}'.format(state_should)) return "\n".join(code) + + def get_args_parser(self): + parser = argparse.ArgumentParser(add_help=False, + argument_default=argparse.SUPPRESS) + parser.add_argument('--state', dest='state', action='store', + required=False, default='present') + for param in ('group', 'mode', 'owner', 'source'): + parser.add_argument('--' + param, dest=param, action='store', + required=False, default=None) + + parser.add_argument("object_id", nargs=1) + return parser diff --git a/cdist/core/code.py b/cdist/core/code.py index 34d5bd97..ae1fddf2 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -22,10 +22,8 @@ # import os -import importlib.util import inspect -import cdist -from cdist.core.pytypes import PythonType +from cdist.core.pytypes import get_pytype_class from . import util @@ -122,25 +120,8 @@ class Code(object): def run_py(self, cdist_object): cdist_type = cdist_object.cdist_type - module_name = cdist_type.name - file_path = os.path.join(cdist_type.absolute_path, '__init__.py') - - if os.path.isfile(file_path): - spec = importlib.util.spec_from_file_location(module_name, - file_path) - m = importlib.util.module_from_spec(spec) - spec.loader.exec_module(m) - classes = inspect.getmembers(m, inspect.isclass) - type_class = None - for _, cl in classes: - if cl != PythonType and issubclass(cl, PythonType): - if type_class: - raise cdist.Error("Only one python type class is " - "supported, but at least two " - "found: {}".format((type_class, - cl, ))) - else: - type_class = cl + type_class = get_pytype_class(cdist_type) + if type_class is not None: env = os.environ.copy() env.update(self.env) message_prefix = cdist_object.name diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 279f5cf9..9b3eb62a 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -22,12 +22,11 @@ import logging import os -import importlib.util import inspect import cdist import cdist.emulator from . import util -from cdist.core.pytypes import PythonType, Command +from cdist.core.pytypes import Command, get_pytype_class ''' @@ -234,27 +233,11 @@ class Manifest(object): def run_py_type_manifest(self, cdist_object): cdist_type = cdist_object.cdist_type - module_name = cdist_type.name - file_path = os.path.join(cdist_type.absolute_path, '__init__.py') - message_prefix = cdist_object.name - if os.path.isfile(file_path): + type_class = get_pytype_class(cdist_type) + if type_class is not None: self.log.verbose("Running python type manifest for object %s", cdist_object.name) - spec = importlib.util.spec_from_file_location(module_name, - file_path) - m = importlib.util.module_from_spec(spec) - spec.loader.exec_module(m) - classes = inspect.getmembers(m, inspect.isclass) - type_class = None - for _, cl in classes: - if cl != PythonType and issubclass(cl, PythonType): - if type_class: - raise cdist.Error("Only one python type class is " - "supported, but at least two " - "found: {}".format((type_class, - cl, ))) - else: - type_class = cl + message_prefix = cdist_object.name env = self.env_py_type_manifest(cdist_object) type_obj = type_class(env=env, cdist_object=cdist_object, local=self.local, remote=None, diff --git a/cdist/core/pytypes.py b/cdist/core/pytypes.py index ae4164ec..9dfc8df9 100644 --- a/cdist/core/pytypes.py +++ b/cdist/core/pytypes.py @@ -4,6 +4,9 @@ import io import sys import re from cdist import message, Error +import importlib.util +import inspect +import cdist __all__ = ["PythonType", "Command", "command"] @@ -13,18 +16,20 @@ class PythonType: def __init__(self, env, cdist_object, local, remote, message_prefix=None): self.env = env self.cdist_object = cdist_object - self.object_id = cdist_object.object_id - self.object_name = cdist_object.name - self.cdist_type = cdist_object.cdist_type self.local = local self.remote = remote - self.object_path = cdist_object.absolute_path - self.type_path = cdist_object.cdist_type.absolute_path - self.explorer_path = os.path.join(self.object_path, 'explorer') - self.parameters = cdist_object.parameters - self.stdin_path = os.path.join(self.object_path, 'stdin') - self.log = logging.getLogger( - self.local.target_host[0] + ':' + self.object_name) + if self.cdist_object: + self.object_id = cdist_object.object_id + self.object_name = cdist_object.name + self.cdist_type = cdist_object.cdist_type + self.object_path = cdist_object.absolute_path + self.explorer_path = os.path.join(self.object_path, 'explorer') + self.type_path = cdist_object.cdist_type.absolute_path + self.parameters = cdist_object.parameters + self.stdin_path = os.path.join(self.object_path, 'stdin') + if self.local: + self.log = logging.getLogger( + self.local.target_host[0] + ':' + self.object_name) self.message_prefix = message_prefix self.message = None @@ -62,12 +67,6 @@ class PythonType: def die(self, msg): raise Error("{}: {}".format(self.cdist_object, msg)) - def type_manifest(self): - pass - - def type_gencode(self): - pass - def manifest(self, stdout=None, stderr=None): try: if self.message_prefix: @@ -123,6 +122,15 @@ class PythonType: return match return None + def get_args_parser(self): + pass + + def type_manifest(self): + pass + + def type_gencode(self): + pass + class Command: def __init__(self, name, *args, **kwargs): @@ -160,3 +168,23 @@ class Command: def command(name, *args, **kwargs): return Command(name, *args, **kwargs) + + +def get_pytype_class(cdist_type): + module_name = cdist_type.name + file_path = os.path.join(cdist_type.absolute_path, '__init__.py') + type_class = None + if os.path.isfile(file_path): + spec = importlib.util.spec_from_file_location(module_name, file_path) + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + classes = inspect.getmembers(m, inspect.isclass) + for _, cl in classes: + if cl != PythonType and issubclass(cl, PythonType): + if type_class: + raise cdist.Error( + "Only one python type class is supported, but at least" + " two found: {}".format((type_class, cl, ))) + else: + type_class = cl + return type_class diff --git a/cdist/emulator.py b/cdist/emulator.py index 3cf82f84..e68f5d39 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -29,6 +29,9 @@ import sys import cdist from cdist import core from cdist import flock +import cdist.util.python_type_util as pytype_util +from cdist.core.pytypes import get_pytype_class +import inspect class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -94,7 +97,28 @@ class Emulator(object): def run(self): """Emulate type commands (i.e. __file and co)""" - self.commandline() + args_parser = None + if pytype_util.is_python_type(self.cdist_type): + type_class = get_pytype_class(self.cdist_type) + if type_class is not None: + # We only need to call parse_args so we need plain instance. + type_obj = type_class(env=None, cdist_object=None, local=None, + remote=None) + if (hasattr(type_obj, 'get_args_parser') and + inspect.ismethod(type_obj.get_args_parser)): + args_parser = type_obj.get_args_parser() + self.log.trace("Using python type argument parser") + print("Using python type argument parser") + if args_parser is None: + # Fallback to classic way. + args_parser = self.get_args_parser() + self.log.trace("Fallback to classic argument parser") + print("Fallback to classic argument parser") + else: + args_parser = self.get_args_parser() + self.log.trace("Using emulator classic argument parser") + print("Using emulator classic argument parser") + self.commandline(args_parser) self.init_object() # locking for parallel execution @@ -125,9 +149,7 @@ class Emulator(object): self.log = logging.getLogger(self.target_host[0]) - def commandline(self): - """Parse command line""" - + def get_args_parser(self): parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS) @@ -159,8 +181,10 @@ class Emulator(object): # If not singleton support one positional parameter if not self.cdist_type.is_singleton: parser.add_argument("object_id", nargs=1) + return parser - # And finally parse/verify parameter + def commandline(self, parser): + """Parse command line""" self.args = parser.parse_args(self.argv[1:]) self.log.trace('Args: %s' % self.args) diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 3a4a0b13..388c3caf 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -542,6 +542,9 @@ in shell, since this is the code that is directly executed at target host. When writing python type you can extend **cdist.core.pytypes.PythonType** class. You need to implement the following methods: +* **get_args_parser**: implementation should return **argparse.ArgumentParser** and if + it is undefined or returned None then cdist falls back to classic type parameter + definition and argument parsing * **type_manifest**: implementation should yield **cdist.core.pytypes.** attribute function call result, or **yield from ()** if type does not use other types * **type_gencode**: implementation should return a string consisting of lines From bd1bda84d5d741b99f69f0cb4dd40f35c8d04352 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Sep 2019 20:16:55 +0200 Subject: [PATCH 09/12] Make python file type default, mv old to __file_old --- .../type/{__file_py => __file}/__init__.py | 0 cdist/conf/type/__file/explorer/stat | 60 +++---------- cdist/conf/type/__file/parameter/optional | 1 - .../{__file_py => __file_old}/explorer/cksum | 0 cdist/conf/type/__file_old/explorer/stat | 88 +++++++++++++++++++ .../{__file_py => __file_old}/explorer/type | 0 .../type/{__file => __file_old}/gencode-local | 0 .../{__file => __file_old}/gencode-remote | 0 .../conf/type/{__file => __file_old}/man.rst | 0 .../parameter/default/state | 0 .../parameter/optional | 1 + cdist/conf/type/__file_py/explorer/stat | 56 ------------ 12 files changed, 103 insertions(+), 103 deletions(-) rename cdist/conf/type/{__file_py => __file}/__init__.py (100%) rename cdist/conf/type/{__file_py => __file_old}/explorer/cksum (100%) create mode 100755 cdist/conf/type/__file_old/explorer/stat rename cdist/conf/type/{__file_py => __file_old}/explorer/type (100%) rename cdist/conf/type/{__file => __file_old}/gencode-local (100%) rename cdist/conf/type/{__file => __file_old}/gencode-remote (100%) rename cdist/conf/type/{__file => __file_old}/man.rst (100%) rename cdist/conf/type/{__file_py => __file_old}/parameter/default/state (100%) rename cdist/conf/type/{__file_py => __file_old}/parameter/optional (76%) delete mode 100755 cdist/conf/type/__file_py/explorer/stat diff --git a/cdist/conf/type/__file_py/__init__.py b/cdist/conf/type/__file/__init__.py similarity index 100% rename from cdist/conf/type/__file_py/__init__.py rename to cdist/conf/type/__file/__init__.py diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 13c1c208..8a917556 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -1,7 +1,6 @@ #!/bin/sh # # 2013 Steven Armstrong (steven-cdist armstrong.cc) -# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -26,56 +25,25 @@ destination="/$__object_id" os=$("$__explorer/os") case "$os" in - "freebsd"|"netbsd"|"openbsd"|"macosx") - stat -f "type: %HT + "freebsd"|"netbsd"|"openbsd") + # FIXME: should be something like this based on man page, but can not test + stat -f "type: %ST +owner: %Du %Su +group: %Dg %Sg +mode: %Op %Sp +size: %Dz +links: %Dl +" "$destination" + ;; + "macosx") + stat -f "type: %HT owner: %Du %Su group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl -" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' - ;; - alpine) - # busybox stat - stat -c "type: %F -owner: %u %U -group: %g %G -mode: %a %A -size: %s -links: %h " "$destination" - ;; - solaris) - ls1="$( ls -ld "$destination" )" - ls2="$( ls -ldn "$destination" )" - - if [ -f "$__object/parameter/mode" ] - then mode_should="$( cat "$__object/parameter/mode" )" - fi - - # yes, it is ugly hack, but if you know better way... - if [ -z "$( find "$destination" -perm "$mode_should" )" ] - then octets=888 - else octets="$( echo "$mode_should" | sed 's/^0//' )" - fi - - case "$( echo "$ls1" | cut -c1-1 )" in - -) echo 'type: regular file' ;; - d) echo 'type: directory' ;; - esac - - echo "owner: $( echo "$ls2" \ - | awk '{print $3}' ) $( echo "$ls1" \ - | awk '{print $3}' )" - - echo "group: $( echo "$ls2" \ - | awk '{print $4}' ) $( echo "$ls1" \ - | awk '{print $4}' )" - - echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" - echo "size: $( echo "$ls1" | awk '{print $5}' )" - echo "links: $( echo "$ls1" | awk '{print $2}' )" - ;; + ;; *) stat --printf="type: %F owner: %u %U @@ -84,5 +52,5 @@ mode: %a %A size: %s links: %h " "$destination" - ;; + ;; esac diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional index 9b98352c..c696d592 100644 --- a/cdist/conf/type/__file/parameter/optional +++ b/cdist/conf/type/__file/parameter/optional @@ -3,4 +3,3 @@ group mode owner source -onchange diff --git a/cdist/conf/type/__file_py/explorer/cksum b/cdist/conf/type/__file_old/explorer/cksum similarity index 100% rename from cdist/conf/type/__file_py/explorer/cksum rename to cdist/conf/type/__file_old/explorer/cksum diff --git a/cdist/conf/type/__file_old/explorer/stat b/cdist/conf/type/__file_old/explorer/stat new file mode 100755 index 00000000..13c1c208 --- /dev/null +++ b/cdist/conf/type/__file_old/explorer/stat @@ -0,0 +1,88 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +# nothing to work with, nothing we could do +[ -e "$destination" ] || exit 0 + +os=$("$__explorer/os") +case "$os" in + "freebsd"|"netbsd"|"openbsd"|"macosx") + stat -f "type: %HT +owner: %Du %Su +group: %Dg %Sg +mode: %Lp %Sp +size: %Dz +links: %Dl +" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' + ;; + alpine) + # busybox stat + stat -c "type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" + echo "size: $( echo "$ls1" | awk '{print $5}' )" + echo "links: $( echo "$ls1" | awk '{print $2}' )" + ;; + *) + stat --printf="type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; +esac diff --git a/cdist/conf/type/__file_py/explorer/type b/cdist/conf/type/__file_old/explorer/type similarity index 100% rename from cdist/conf/type/__file_py/explorer/type rename to cdist/conf/type/__file_old/explorer/type diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file_old/gencode-local similarity index 100% rename from cdist/conf/type/__file/gencode-local rename to cdist/conf/type/__file_old/gencode-local diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file_old/gencode-remote similarity index 100% rename from cdist/conf/type/__file/gencode-remote rename to cdist/conf/type/__file_old/gencode-remote diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file_old/man.rst similarity index 100% rename from cdist/conf/type/__file/man.rst rename to cdist/conf/type/__file_old/man.rst diff --git a/cdist/conf/type/__file_py/parameter/default/state b/cdist/conf/type/__file_old/parameter/default/state similarity index 100% rename from cdist/conf/type/__file_py/parameter/default/state rename to cdist/conf/type/__file_old/parameter/default/state diff --git a/cdist/conf/type/__file_py/parameter/optional b/cdist/conf/type/__file_old/parameter/optional similarity index 76% rename from cdist/conf/type/__file_py/parameter/optional rename to cdist/conf/type/__file_old/parameter/optional index c696d592..9b98352c 100644 --- a/cdist/conf/type/__file_py/parameter/optional +++ b/cdist/conf/type/__file_old/parameter/optional @@ -3,3 +3,4 @@ group mode owner source +onchange diff --git a/cdist/conf/type/__file_py/explorer/stat b/cdist/conf/type/__file_py/explorer/stat deleted file mode 100755 index 8a917556..00000000 --- a/cdist/conf/type/__file_py/explorer/stat +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# 2013 Steven Armstrong (steven-cdist armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -destination="/$__object_id" - -# nothing to work with, nothing we could do -[ -e "$destination" ] || exit 0 - -os=$("$__explorer/os") -case "$os" in - "freebsd"|"netbsd"|"openbsd") - # FIXME: should be something like this based on man page, but can not test - stat -f "type: %ST -owner: %Du %Su -group: %Dg %Sg -mode: %Op %Sp -size: %Dz -links: %Dl -" "$destination" - ;; - "macosx") - stat -f "type: %HT -owner: %Du %Su -group: %Dg %Sg -mode: %Lp %Sp -size: %Dz -links: %Dl -" "$destination" - ;; - *) - stat --printf="type: %F -owner: %u %U -group: %g %G -mode: %a %A -size: %s -links: %h -" "$destination" - ;; -esac From 07638e662877c85cf6b604a64b6f9aeb3e656212 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Sep 2019 20:24:45 +0200 Subject: [PATCH 10/12] Take stat from latest shell __file type --- cdist/conf/type/__file/explorer/stat | 60 +++++++++++++++++++++------- 1 file changed, 46 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 8a917556..13c1c208 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -1,6 +1,7 @@ #!/bin/sh # # 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -25,25 +26,56 @@ destination="/$__object_id" os=$("$__explorer/os") case "$os" in - "freebsd"|"netbsd"|"openbsd") - # FIXME: should be something like this based on man page, but can not test - stat -f "type: %ST -owner: %Du %Su -group: %Dg %Sg -mode: %Op %Sp -size: %Dz -links: %Dl -" "$destination" - ;; - "macosx") - stat -f "type: %HT + "freebsd"|"netbsd"|"openbsd"|"macosx") + stat -f "type: %HT owner: %Du %Su group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl +" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' + ;; + alpine) + # busybox stat + stat -c "type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h " "$destination" - ;; + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" + echo "size: $( echo "$ls1" | awk '{print $5}' )" + echo "links: $( echo "$ls1" | awk '{print $2}' )" + ;; *) stat --printf="type: %F owner: %u %U @@ -52,5 +84,5 @@ mode: %a %A size: %s links: %h " "$destination" - ;; + ;; esac From 2831ec00903d42ef6b5000c69f78146a100da135 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 1 Oct 2019 08:14:24 +0200 Subject: [PATCH 11/12] Remove unnecessary files --- cdist/conf/type/__file/parameter/default/state | 1 - cdist/conf/type/__file/parameter/optional | 5 ----- 2 files changed, 6 deletions(-) delete mode 100644 cdist/conf/type/__file/parameter/default/state delete mode 100644 cdist/conf/type/__file/parameter/optional diff --git a/cdist/conf/type/__file/parameter/default/state b/cdist/conf/type/__file/parameter/default/state deleted file mode 100644 index e7f6134f..00000000 --- a/cdist/conf/type/__file/parameter/default/state +++ /dev/null @@ -1 +0,0 @@ -present diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional deleted file mode 100644 index c696d592..00000000 --- a/cdist/conf/type/__file/parameter/optional +++ /dev/null @@ -1,5 +0,0 @@ -state -group -mode -owner -source From 7c52d4c28c4e63c7bc9ab38b92f08918cc8e0188 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 1 Oct 2019 08:15:01 +0200 Subject: [PATCH 12/12] Align with shell type: implement onchange --- cdist/conf/type/__file/__init__.py | 11 ++++++++++- cdist/conf/type/__file_py | 1 + 2 files changed, 11 insertions(+), 1 deletion(-) create mode 120000 cdist/conf/type/__file_py diff --git a/cdist/conf/type/__file/__init__.py b/cdist/conf/type/__file/__init__.py index 3397b417..bcb525fc 100644 --- a/cdist/conf/type/__file/__init__.py +++ b/cdist/conf/type/__file/__init__.py @@ -52,6 +52,7 @@ class FileType(PythonType): create_file = False upload_file = False set_attributes = False + fire_onchange = False code = [] if state_should == 'present' or state_should == 'exists': if source is None: @@ -72,6 +73,7 @@ class FileType(PythonType): self.die('Source {} does not exist'.format(source)) if create_file or upload_file: set_attributes = True + fire_onchange = True tempfile_template = '{}.cdist.XXXXXXXXXX'.format(destination) destination_upload = self.run_remote( ["mktemp", tempfile_template, ]) @@ -90,6 +92,7 @@ class FileType(PythonType): value_is = self.get_attribute(stat_file, attribute, value_should) if set_attributes or value_should != value_is: + fire_onchange = True code.append(self.set_attribute(attribute, value_should, destination)) @@ -97,9 +100,15 @@ class FileType(PythonType): if typeis == 'file': code.append('rm -f {}'.format(destination)) self.send_message('remove') + fire_onchange = True else: self.die('Unknown state {}'.format(state_should)) + if fire_onchange: + onchange = self.get_parameter('onchange') + if onchange: + code.append(onchange) + return "\n".join(code) def get_args_parser(self): @@ -107,7 +116,7 @@ class FileType(PythonType): argument_default=argparse.SUPPRESS) parser.add_argument('--state', dest='state', action='store', required=False, default='present') - for param in ('group', 'mode', 'owner', 'source'): + for param in ('group', 'mode', 'owner', 'source', 'onchange'): parser.add_argument('--' + param, dest=param, action='store', required=False, default=None) diff --git a/cdist/conf/type/__file_py b/cdist/conf/type/__file_py new file mode 120000 index 00000000..efa910bd --- /dev/null +++ b/cdist/conf/type/__file_py @@ -0,0 +1 @@ +__file \ No newline at end of file