From 29c0180204beced2813f2d1ec2b50ee069d205c6 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Thu, 28 May 2020 23:31:13 +0300 Subject: [PATCH 01/37] [__ssh_authorized_keys] add --remove-unknown parameter --- .../type/__ssh_authorized_keys/explorer/keys | 8 ++++++ cdist/conf/type/__ssh_authorized_keys/man.rst | 9 +++++++ .../conf/type/__ssh_authorized_keys/manifest | 27 ++++++++++++++++++- .../__ssh_authorized_keys/parameter/boolean | 1 + 4 files changed, 44 insertions(+), 1 deletion(-) create mode 100755 cdist/conf/type/__ssh_authorized_keys/explorer/keys diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys new file mode 100755 index 00000000..b515c630 --- /dev/null +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -0,0 +1,8 @@ +#!/bin/sh -e + +file="$( . "$__type_explorer/file" )" + +if [ -f "$file" ] +then + cat "$file" +fi diff --git a/cdist/conf/type/__ssh_authorized_keys/man.rst b/cdist/conf/type/__ssh_authorized_keys/man.rst index 93357b1d..dac6adeb 100644 --- a/cdist/conf/type/__ssh_authorized_keys/man.rst +++ b/cdist/conf/type/__ssh_authorized_keys/man.rst @@ -60,6 +60,9 @@ nofile Don't manage existence, ownership and permissions of the the authorized_keys file. +remove-unknown + Remove undefined keys. + EXAMPLES -------- @@ -70,6 +73,12 @@ EXAMPLES __ssh_authorized_keys root \ --key "$(cat ~/.ssh/id_rsa.pub)" + # same as above, but make sure your key is only key in + # root's authorized_keys file + __ssh_authorized_keys root \ + --key "$(cat ~/.ssh/id_rsa.pub)" \ + --remove-unknown + # allow key to login as user-name __ssh_authorized_keys user-name \ --key "ssh-rsa AXYZAAB3NzaC1yc2..." diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest index b9f0582e..b319316b 100755 --- a/cdist/conf/type/__ssh_authorized_keys/manifest +++ b/cdist/conf/type/__ssh_authorized_keys/manifest @@ -55,8 +55,12 @@ _cksum() { echo "$1" | cksum | cut -d' ' -f 1 } +_type_and_key() { + echo "$1" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }' +} + while read -r key; do - type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" + type_and_key="$( _type_and_key "$key" )" object_id="$(_cksum "$file")-$(_cksum "$type_and_key")" set -- "$object_id" set -- "$@" --file "$file" @@ -72,3 +76,24 @@ while read -r key; do # Ensure __ssh_authorized_key does not read stdin __ssh_authorized_key "$@" < /dev/null done < "$__object/parameter/key" + +if [ -f "$__object/parameter/remove-unknown" ] && + [ -s "$__object/explorer/keys" ] +then + while read -r key + do + type_and_key="$( _type_and_key "$key" )" + + if grep -Fq "$type_and_key" "$__object/parameter/key" + then + continue + fi + + __ssh_authorized_key "remove-$( _cksum "$file$key" )" \ + --file "$file" \ + --key "$key" \ + --state absent \ + < /dev/null + done \ + < "$__object/explorer/keys" +fi diff --git a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean index 4bb126fe..7388fed5 100644 --- a/cdist/conf/type/__ssh_authorized_keys/parameter/boolean +++ b/cdist/conf/type/__ssh_authorized_keys/parameter/boolean @@ -1,2 +1,3 @@ noparent nofile +remove-unknown From b354ea6e9430816bd62168c4c8e7b936ad4a4885 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 30 May 2020 10:49:13 +0200 Subject: [PATCH 02/37] ++changelog --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index ad008cb2..76b5edfa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) + 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) * Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius) From 988190363aeff5fba9be80355c318e1ff372eda9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 30 May 2020 15:10:13 +0200 Subject: [PATCH 03/37] Resolve shellcheck SC1090 --- cdist/conf/type/__ssh_authorized_keys/explorer/keys | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__ssh_authorized_keys/explorer/keys b/cdist/conf/type/__ssh_authorized_keys/explorer/keys index b515c630..cec25746 100755 --- a/cdist/conf/type/__ssh_authorized_keys/explorer/keys +++ b/cdist/conf/type/__ssh_authorized_keys/explorer/keys @@ -1,5 +1,6 @@ #!/bin/sh -e +# shellcheck disable=SC1090 file="$( . "$__type_explorer/file" )" if [ -f "$file" ] From ba77ea9edcf2d99c6edd76adaa83017821babae3 Mon Sep 17 00:00:00 2001 From: Evil Ham Date: Mon, 1 Jun 2020 19:11:58 +0200 Subject: [PATCH 04/37] [UX] Add option to enable LogLevel-based coloured output. This makes it easier for new and experienced users to run cdist with higher verbosity levels, both to know that things are working as expected and to debug issues. Documentation has been modified accordingly and default behaviour is not changed. --- cdist/argparse.py | 31 ++++++++++++++++-- cdist/configuration.py | 11 +++++++ cdist/core/manifest.py | 1 + cdist/emulator.py | 3 ++ cdist/log.py | 26 ++++++++++++++- cdist/test/configuration/__init__.py | 28 +++++++++++++++++ configuration/cdist.cfg.skeleton | 6 ++++ docs/src/cdist-reference.rst.sh | 5 +++ docs/src/man1/cdist.rst | 47 +++++++++++++++++++--------- 9 files changed, 140 insertions(+), 18 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 611c484a..c30e2030 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -5,6 +5,7 @@ import logging import collections import functools import cdist.configuration +import cdist.log import cdist.preos import cdist.info @@ -88,6 +89,13 @@ def check_lower_bounded_int(value, lower_bound, name): return val +def colored_output_type(val): + boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES + if val not in boolean_states.keys(): + raise argparse.ArgumentError() + return boolean_states[val] + + def get_parsers(): global parser @@ -125,6 +133,15 @@ def get_parsers(): 'value.'), action='count', default=None) + parser['colored_output'] = argparse.ArgumentParser(add_help=False) + parser['colored_output'].add_argument( + '--colors', + help='Use a colored output for different log levels.' + 'It can be a boolean or "auto" (default) which enables this ' + 'feature if stdout is a tty and disables it otherwise.', + action='store', dest='colored_output', required=False, + type=colored_output_type) + parser['beta'] = argparse.ArgumentParser(add_help=False) parser['beta'].add_argument( '-b', '--beta', @@ -283,6 +300,7 @@ def get_parsers(): 'host', nargs='*', help='Host(s) to operate on.') parser['config'] = parser['sub'].add_parser( 'config', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['config_main'], parser['inventory_common'], @@ -301,6 +319,7 @@ def get_parsers(): parser['add-host'] = parser['invsub'].add_parser( 'add-host', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['add-host'].add_argument( @@ -315,6 +334,7 @@ def get_parsers(): parser['add-tag'] = parser['invsub'].add_parser( 'add-tag', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['add-tag'].add_argument( @@ -346,6 +366,7 @@ def get_parsers(): parser['del-host'] = parser['invsub'].add_parser( 'del-host', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['del-host'].add_argument( @@ -363,6 +384,7 @@ def get_parsers(): parser['del-tag'] = parser['invsub'].add_parser( 'del-tag', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['del-tag'].add_argument( @@ -398,6 +420,7 @@ def get_parsers(): parser['list'] = parser['invsub'].add_parser( 'list', parents=[parser['loglevel'], parser['beta'], + parser['colored_output'], parser['common'], parser['inventory_common']]) parser['list'].add_argument( @@ -430,7 +453,7 @@ def get_parsers(): # Shell parser['shell'] = parser['sub'].add_parser( - 'shell', parents=[parser['loglevel']]) + 'shell', parents=[parser['loglevel'], parser['colored_output']]) parser['shell'].add_argument( '-s', '--shell', help=('Select shell to use, defaults to current shell. Used shell' @@ -495,9 +518,13 @@ def parse_and_configure(argv, singleton=True): log = logging.getLogger("cdist") + config = cfg.get_config() + if config.get('GLOBAL', {}).get('colored_output', False): + cdist.log.ColorFormatter.USE_COLORS = True + log.verbose("version %s" % cdist.VERSION) log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(cfg.get_config())) + log.trace('configuration: {}'.format(config)) log.trace('configured args: {}'.format(args)) check_beta(vars(args)) diff --git a/cdist/configuration.py b/cdist/configuration.py index 1011a382..6f07c27f 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -27,6 +27,7 @@ import cdist.argparse import re import multiprocessing import logging +import sys class Singleton(type): @@ -47,6 +48,7 @@ _VERBOSITY_VALUES = ( _ARCHIVING_VALUES = ( 'tar', 'tgz', 'tbz2', 'txz', 'none', ) +_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty() class OptionBase: @@ -246,9 +248,15 @@ class LogLevelOption(OptionBase): return VerbosityOption().translate(val) +class ColoredOutputOption(BooleanOption): + BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES, + auto=_COLORED_OUTPUT_DEFAULT) + + _ARG_OPTION_MAPPING = { 'beta': 'beta', 'cache_path_pattern': 'cache_path_pattern', + 'colored_output': 'colored_output', 'conf_dir': 'conf_dir', 'manifest': 'init_manifest', 'out_path': 'out_path', @@ -294,6 +302,7 @@ class Configuration(metaclass=Singleton): 'remote_shell': StringOption('remote_shell'), 'cache_path_pattern': StringOption('cache_path_pattern'), 'conf_dir': ConfDirOption(), + 'colored_output': ColoredOutputOption('colored_output'), 'init_manifest': StringOption('init_manifest'), 'out_path': StringOption('out_path'), 'remote_out_path': StringOption('remote_out_path'), @@ -319,6 +328,7 @@ class Configuration(metaclass=Singleton): 'CDIST_REMOTE_COPY': 'remote_copy', 'CDIST_INVENTORY_DIR': 'inventory_dir', 'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern', + 'CDIST_COLORED_OUTPUT': 'colored_output', '__cdist_log_level': 'verbosity', } ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', ) @@ -332,6 +342,7 @@ class Configuration(metaclass=Singleton): } REQUIRED_DEFAULT_CONFIG_VALUES = { 'GLOBAL': { + 'colored_output': _COLORED_OUTPUT_DEFAULT, 'verbosity': 0, }, } diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8aeaf860..32520e49 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -119,6 +119,7 @@ class Manifest(object): '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( self.log), + '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS), } if dry_run: diff --git a/cdist/emulator.py b/cdist/emulator.py index 4800e2a3..87c9fe12 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -129,6 +129,9 @@ class Emulator(object): # if invalid __cdist_log_level value logging.root.setLevel(logging.WARNING) + colored_log = self.env.get('__cdist_colored_log', 'False') + cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True' + self.log = logging.getLogger(self.target_host[0]) def commandline(self): diff --git a/cdist/log.py b/cdist/log.py index 2d0bef0b..5f2d8f53 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -50,6 +50,30 @@ def _trace(msg, *args, **kwargs): logging.trace = _trace +class ColorFormatter(logging.Formatter): + USE_COLORS = False + RESET = '\033[0m' + COLOR_MAP = { + 'ERROR': '\033[0;31m', + 'WARNING': '\033[0;33m', + 'INFO': '\033[0;94m', + 'VERBOSE': '\033[0;34m', + 'DEBUG': '\033[0;90m', + 'TRACE': '\033[0;37m', + } + + def __init__(self, msg): + super().__init__(msg) + + def format(self, record): + msg = super().format(record) + if self.USE_COLORS: + color = self.COLOR_MAP.get(record.levelname) + if color: + msg = color + msg + self.RESET + return msg + + class DefaultLog(logging.Logger): FORMAT = '%(levelname)s: %(message)s' @@ -66,7 +90,7 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - formatter = logging.Formatter(self.FORMAT) + formatter = ColorFormatter(self.FORMAT) self.addFilter(self) diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 182868a6..07a73bda 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -28,10 +28,12 @@ import argparse from cdist import test import cdist.argparse as cap import logging +import sys my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") +colored_output_default = sys.stdout.isatty() def newConfigParser(): @@ -153,6 +155,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': '', 'cache_path_pattern': '', + 'colored_output': colored_output_default, 'conf_dir': '', 'init_manifest': '', 'out_path': '', @@ -184,6 +187,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': None, 'init_manifest': None, 'out_path': None, @@ -390,6 +394,7 @@ class ConfigurationTestCase(test.CdistTestCase): args = argparse.Namespace() expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'verbosity': 0, }, } @@ -440,6 +445,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': None, 'init_manifest': None, 'out_path': None, @@ -515,6 +521,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': ['/opt/cdist', ], 'init_manifest': None, 'out_path': None, @@ -556,6 +563,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': '', 'cache_path_pattern': '', + 'colored_output': colored_output_default, 'conf_dir': '', 'init_manifest': '', 'out_path': '', @@ -579,6 +587,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -623,6 +632,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': '', 'cache_path_pattern': '', + 'colored_output': colored_output_default, 'conf_dir': '', 'init_manifest': '', 'out_path': '', @@ -645,6 +655,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'local_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', + 'colored_output': colored_output_default, 'conf_dir': '/opt/cdist', 'remote_copy': 'myscp', 'remote_exec': 'myexec', @@ -663,6 +674,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -694,6 +706,7 @@ class ConfigurationTestCase(test.CdistTestCase): } expected_config = { 'GLOBAL': { + 'colored_output': colored_output_default, 'verbosity': 0, }, } @@ -767,6 +780,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/opt/sysadmin/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -865,6 +879,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -964,6 +979,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -1063,6 +1079,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -1095,6 +1112,7 @@ class ConfigurationTestCase(test.CdistTestCase): 'beta': True, 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, + 'colored_output': colored_output_default, 'conf_dir': [ '/opt/conf/cdist', ], @@ -1125,6 +1143,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { 'inventory_dir': None, + 'colored_output': colored_output_default, 'conf_dir': None, 'verbosity': 0, }, @@ -1148,6 +1167,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'verbosity': cap.VERBOSE_DEBUG, }, } @@ -1185,6 +1205,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': True, 'verbosity': 0, }, @@ -1213,6 +1234,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': False, 'verbosity': 0, }, @@ -1241,6 +1263,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': False, 'verbosity': 0, }, @@ -1269,6 +1292,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'save_output_streams': False, 'verbosity': 0, }, @@ -1308,6 +1332,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': True, 'verbosity': 0, }, @@ -1336,6 +1361,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': True, 'verbosity': 0, }, @@ -1364,6 +1390,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': False, 'verbosity': 0, }, @@ -1392,6 +1419,7 @@ class ConfigurationTestCase(test.CdistTestCase): expected_config_dict = { 'GLOBAL': { + 'colored_output': colored_output_default, 'timestamp': False, 'verbosity': 0, }, diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 91c5ab02..f2a09064 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -13,6 +13,12 @@ # Specify cache path pattern. # cache_path_pattern = %h # +# colored_output +# Use a colored output for different log levels. +# It can be a boolean or 'auto' (default) which enables this feature if +# stdout is a tty and disables it otherwise. +# colored_output = auto +# # conf_dir # List of configuration directories separated with the character conventionally # used by the operating system to separate search path components (as in PATH), diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index e77d98f6..3b997f63 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -344,6 +344,11 @@ CDIST_INVENTORY_DIR CDIST_BETA Enable beta functionalities. +CDIST_COLORED_OUTPUT + Use a colored output for different log levels. + It can be a boolean or 'auto' (default) which enables this feature if + stdout is a tty and disables it otherwise. + CDIST_CACHE_PATH_PATTERN Custom cache path pattern. eof diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 38248821..4c34c4b7 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -15,8 +15,9 @@ SYNOPSIS cdist banner [-h] [-l LOGLEVEL] [-q] [-v] - cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] - [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] + [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] @@ -24,8 +25,9 @@ SYNOPSIS [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] - [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] + [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] @@ -36,26 +38,29 @@ SYNOPSIS cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] - [-f HOSTFILE] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-f HOSTFILE] [host [host ...]] cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] - [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] + [-t TAGLIST] [host [host ...]] cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [host [host ...]] cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + [-T TAGFILE] [-t TAGLIST] [host [host ...]] - cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] + cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] [host [host ...]] @@ -84,9 +89,11 @@ SYNOPSIS [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] target_dir - cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] + cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT] + [-s SHELL] - cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] + cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] + [pattern] DESCRIPTION @@ -104,6 +111,11 @@ All commands accept the following options: **-h, --help** Show the help screen. +**--colors COLORED_OUTPUT** + Use a colored output for different log levels.It can + be a boolean or "auto" (default) which enables this + feature if stdout is a tty and disables it otherwise. + **-l LOGLEVEL, --log-level LOGLEVEL** Set the specified verbosity level. The levels, in order from the lowest to the highest, are: ERROR (-1), @@ -893,6 +905,11 @@ CDIST_BETA CDIST_CACHE_PATH_PATTERN Custom cache path pattern. +CDIST_COLORED_OUTPUT + Use a colored output for different log levels. + It can be a boolean or 'auto' (default) which enables this feature if + stdout is a tty and disables it otherwise. + CDIST_CONFIG_FILE Custom configuration file. From 9a4e3488c2104ad5562b5ab9aad8120969df73a8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 1 Jun 2020 19:17:02 +0200 Subject: [PATCH 05/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 76b5edfa..5408c54f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,7 @@ Changelog next: * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) + * Core: Support colored log output (Evil Ham) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 55ebd1a4c56e788b18911f273d931f950a0c5271 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 1 Jun 2020 20:22:40 +0200 Subject: [PATCH 06/37] Fix man build failure in newer sphinx versions --- cdist/sphinxext/manpage.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py index 135fe22e..1f8ac4f6 100644 --- a/cdist/sphinxext/manpage.py +++ b/cdist/sphinxext/manpage.py @@ -7,6 +7,7 @@ from docutils.io import FileOutput from os import path from sphinx.util.nodes import inline_all_toctrees from sphinx import addnodes +from sphinx.util import logging """ Extension based on sphinx builtin manpage. @@ -15,6 +16,9 @@ from sphinx import addnodes """ +logger = logging.getLogger(__name__) + + class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): def header(self): @@ -28,7 +32,7 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): def __init__(self, builder): super().__init__(builder) self.translator_class = ( - self.builder.translator_class or ManualPageTranslator) + self.builder.get_translator_class() or ManualPageTranslator) class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): @@ -43,7 +47,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): components=(docwriter,), read_config_files=True).get_default_values() - self.info(bold('writing... '), nonl=True) + logger.info(bold('writing... '), nonl=True) for info in self.config.man_pages: docname, name, description, authors, section = info @@ -54,7 +58,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): authors = [] targetname = '%s.%s' % (name, section) - self.info(darkgreen(targetname) + ' { ', nonl=True) + logger.info(darkgreen(targetname) + ' { ', nonl=True) destination = FileOutput( destination_path=path.join(self.outdir, targetname), encoding='utf-8') @@ -63,7 +67,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): docnames = set() largetree = inline_all_toctrees(self, docnames, docname, tree, darkgreen, [docname]) - self.info('} ', nonl=True) + logger.info('} ', nonl=True) self.env.resolve_references(largetree, docname, self) # remove pending_xref nodes for pendingnode in largetree.traverse(addnodes.pending_xref): @@ -76,7 +80,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): largetree.settings.section = section docwriter.write(largetree, destination) - self.info() + logger.info("") def setup(app): From 48d66b014351982e70aa3e065ffc1ed344660bab Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 1 Jun 2020 22:25:15 +0200 Subject: [PATCH 07/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5408c54f..5b57e98b 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) + * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 6a611e556a54a60952a833989a5a0fca79011217 Mon Sep 17 00:00:00 2001 From: Daniel Heule Date: Tue, 2 Jun 2020 13:40:21 +0200 Subject: [PATCH 08/37] fix os explorer for sles15 --- cdist/conf/explorer/os | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 563fa4cf..2d2aede6 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -143,6 +143,11 @@ case "$uname_s" in esac if [ -f /etc/os-release ]; then + # after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse + if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then + echo suse + exit 0 + fi # already lowercase, according to: # https://www.freedesktop.org/software/systemd/man/os-release.html awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release From 747c6b10762e18bbda24fbb876ac2e88f29dc70b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 3 Jun 2020 21:45:04 +0200 Subject: [PATCH 09/37] Respect NO_COLOR environment variable --- cdist/argparse.py | 23 +++++++++-------------- cdist/config.py | 1 + cdist/configuration.py | 28 +++++++++++++++++++--------- cdist/core/manifest.py | 2 +- cdist/emulator.py | 4 ++-- cdist/log.py | 19 ++++++++++--------- cdist/test/configuration/__init__.py | 2 +- configuration/cdist.cfg.skeleton | 6 ++++-- 8 files changed, 47 insertions(+), 38 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index c30e2030..0782654f 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -89,13 +89,6 @@ def check_lower_bounded_int(value, lower_bound, name): return val -def colored_output_type(val): - boolean_states = cdist.configuration.ColoredOutputOption.BOOLEAN_STATES - if val not in boolean_states.keys(): - raise argparse.ArgumentError() - return boolean_states[val] - - def get_parsers(): global parser @@ -140,7 +133,7 @@ def get_parsers(): 'It can be a boolean or "auto" (default) which enables this ' 'feature if stdout is a tty and disables it otherwise.', action='store', dest='colored_output', required=False, - type=colored_output_type) + choices=cdist.configuration.ColoredOutputOption.CHOICES) parser['beta'] = argparse.ArgumentParser(add_help=False) parser['beta'].add_argument( @@ -501,7 +494,12 @@ def handle_loglevel(args): if hasattr(args, 'quiet') and args.quiet: args.verbose = _verbosity_level_off - logging.root.setLevel(_verbosity_level[args.verbose]) + logging.getLogger().setLevel(_verbosity_level[args.verbose]) + + +def handle_log_colors(args): + if cdist.configuration.ColoredOutputOption.translate(args.colored_output): + cdist.log.DefaultLog.USE_COLORS = True def parse_and_configure(argv, singleton=True): @@ -515,16 +513,13 @@ def parse_and_configure(argv, singleton=True): raise cdist.Error(str(e)) # Loglevels are handled globally in here handle_loglevel(args) + handle_log_colors(args) log = logging.getLogger("cdist") - config = cfg.get_config() - if config.get('GLOBAL', {}).get('colored_output', False): - cdist.log.ColorFormatter.USE_COLORS = True - log.verbose("version %s" % cdist.VERSION) log.trace('command line args: {}'.format(cfg.command_line_args)) - log.trace('configuration: {}'.format(config)) + log.trace('configuration: {}'.format(cfg.get_config())) log.trace('configured args: {}'.format(args)) check_beta(vars(args)) diff --git a/cdist/config.py b/cdist/config.py index 97cc1da6..b2d72f05 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -203,6 +203,7 @@ class Config(object): cdist.log.setupParallelLogging() elif args.timestamp: cdist.log.setupTimestampingLogging() + log = logging.getLogger("config") # No new child process if only one host at a time. diff --git a/cdist/configuration.py b/cdist/configuration.py index 6f07c27f..c0fbc063 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -48,7 +48,6 @@ _VERBOSITY_VALUES = ( _ARCHIVING_VALUES = ( 'tar', 'tgz', 'tbz2', 'txz', 'none', ) -_COLORED_OUTPUT_DEFAULT = sys.stdout.isatty() class OptionBase: @@ -249,8 +248,22 @@ class LogLevelOption(OptionBase): class ColoredOutputOption(BooleanOption): - BOOLEAN_STATES = dict(configparser.ConfigParser.BOOLEAN_STATES, - auto=_COLORED_OUTPUT_DEFAULT) + CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',) + DEFAULT = 'auto' + + def get_converter(self): + return self.translate + + @staticmethod + def translate(val): + if 'NO_COLOR' in os.environ: + return False + elif isinstance(val, bool): + return val + elif val == 'auto': + return sys.stdout.isatty() + else: + return configparser.ConfigParser.BOOLEAN_STATES[val] _ARG_OPTION_MAPPING = { @@ -337,12 +350,10 @@ class Configuration(metaclass=Singleton): } ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING - ADJUST_ARG_OPTION_MAPPING = { - _ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING - } + ADJUST_ARG_OPTION_MAPPING = {v: k for k, v in _ARG_OPTION_MAPPING.items()} REQUIRED_DEFAULT_CONFIG_VALUES = { 'GLOBAL': { - 'colored_output': _COLORED_OUTPUT_DEFAULT, + 'colored_output': 'auto', 'verbosity': 0, }, } @@ -495,8 +506,7 @@ class Configuration(metaclass=Singleton): newconfig = self._read_config_file(config_file) self._update_config_dict(config, newconfig) # command line config file - if (self.args and 'config_file' in self.args and - self.args['config_file']): + if (self.args and self.args.get('config_file', None)): newconfig = self._read_config_file(self.args['config_file']) self._update_config_dict(config, newconfig) # command line diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 32520e49..8b833ff2 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -119,7 +119,7 @@ class Manifest(object): '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( self.log), - '__cdist_colored_log': str(cdist.log.ColorFormatter.USE_COLORS), + '__cdist_colored_log': str(self.log.USE_COLORS).lower(), } if dry_run: diff --git a/cdist/emulator.py b/cdist/emulator.py index 87c9fe12..4eaf2c93 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -129,8 +129,8 @@ class Emulator(object): # if invalid __cdist_log_level value logging.root.setLevel(logging.WARNING) - colored_log = self.env.get('__cdist_colored_log', 'False') - cdist.log.ColorFormatter.USE_COLORS = colored_log == 'True' + colored_log = self.env.get('__cdist_colored_log', 'false') + cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true' self.log = logging.getLogger(self.target_host[0]) diff --git a/cdist/log.py b/cdist/log.py index 5f2d8f53..19efebdb 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -51,7 +51,6 @@ logging.trace = _trace class ColorFormatter(logging.Formatter): - USE_COLORS = False RESET = '\033[0m' COLOR_MAP = { 'ERROR': '\033[0;31m', @@ -62,20 +61,19 @@ class ColorFormatter(logging.Formatter): 'TRACE': '\033[0;37m', } - def __init__(self, msg): - super().__init__(msg) + def __init__(self, fmt): + super().__init__(fmt=fmt) def format(self, record): msg = super().format(record) - if self.USE_COLORS: - color = self.COLOR_MAP.get(record.levelname) - if color: - msg = color + msg + self.RESET + color = self.COLOR_MAP.get(record.levelname) + if color: + msg = color + msg + self.RESET return msg class DefaultLog(logging.Logger): - + USE_COLORS = False FORMAT = '%(levelname)s: %(message)s' class StdoutFilter(logging.Filter): @@ -90,7 +88,10 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - formatter = ColorFormatter(self.FORMAT) + if self.USE_COLORS: + formatter = ColorFormatter(self.FORMAT) + else: + formatter = logging.Formatter(self.FORMAT) self.addFilter(self) diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 07a73bda..5305b6d3 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -33,7 +33,7 @@ import sys my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") -colored_output_default = sys.stdout.isatty() +colored_output_default = 'auto' def newConfigParser(): diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index f2a09064..0730201d 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -17,6 +17,8 @@ # Use a colored output for different log levels. # It can be a boolean or 'auto' (default) which enables this feature if # stdout is a tty and disables it otherwise. +# Colored output is always disabled if the NO_COLOR environment variable is +# defined (https://no-color.org/). # colored_output = auto # # conf_dir @@ -51,7 +53,7 @@ # # out_path # Directory to save cdist output in. -# out_path = +# out_path = # # parallel # Process hosts in parallel. If -1 then the default, number of CPU's in @@ -77,6 +79,6 @@ # remote_shell = /bin/sh # # verbosity -# Set verbosity level. Valid values are: +# Set verbosity level. Valid values are: # ERROR, WARNING, INFO, VERBOSE, DEBUG, TRACE and OFF. # verbosity = INFO From cdb0d2be413114a237a0f56c948cc5cd6cfc2b0b Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Wed, 3 Jun 2020 23:21:50 +0200 Subject: [PATCH 10/37] Patch tests --- cdist/configuration.py | 4 ++++ cdist/test/configuration/__init__.py | 9 ++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cdist/configuration.py b/cdist/configuration.py index c0fbc063..f68a5e09 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -266,6 +266,10 @@ class ColoredOutputOption(BooleanOption): return configparser.ConfigParser.BOOLEAN_STATES[val] +ColoredOutputOption.DEFAULT = ColoredOutputOption.translate( + ColoredOutputOption.DEFAULT) + + _ARG_OPTION_MAPPING = { 'beta': 'beta', 'cache_path_pattern': 'cache_path_pattern', diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 5305b6d3..3fd24ca5 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -187,7 +187,8 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, - 'colored_output': colored_output_default, + 'colored_output': cc.ColoredOutputOption.translate( + colored_output_default), 'conf_dir': None, 'init_manifest': None, 'out_path': None, @@ -587,7 +588,8 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': None, 'cache_path_pattern': None, - 'colored_output': colored_output_default, + 'colored_output': cc.ColoredOutputOption.translate( + colored_output_default), 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', @@ -674,7 +676,8 @@ class ConfigurationTestCase(test.CdistTestCase): 'remote_shell': '/usr/bin/sh', 'inventory_dir': '/var/db/cdist/inventory', 'cache_path_pattern': None, - 'colored_output': colored_output_default, + 'colored_output': cc.ColoredOutputOption.translate( + colored_output_default), 'conf_dir': [ '/opt/cdist/conf', '/usr/local/share/cdist/conf', From 89e48734bf13446024e9009bce7e77cbd4255840 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 5 Jun 2020 12:22:49 +0200 Subject: [PATCH 11/37] Let config file and command line override NO_COLOR envvar --- cdist/configuration.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cdist/configuration.py b/cdist/configuration.py index f68a5e09..a12940b0 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -256,14 +256,12 @@ class ColoredOutputOption(BooleanOption): @staticmethod def translate(val): - if 'NO_COLOR' in os.environ: - return False - elif isinstance(val, bool): + if isinstance(val, bool): return val - elif val == 'auto': - return sys.stdout.isatty() - else: + elif val in configparser.ConfigParser.BOOLEAN_STATES: return configparser.ConfigParser.BOOLEAN_STATES[val] + elif val == 'auto': + return 'NO_COLOR' not in os.environ and sys.stdout.isatty() ColoredOutputOption.DEFAULT = ColoredOutputOption.translate( From 790c6efae9e379cb140e59469f07d2c3732f536c Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 5 Jun 2020 13:56:30 +0200 Subject: [PATCH 12/37] Update colored output documentation --- docs/src/cdist-reference.rst.sh | 38 +++++++++++++++++++++++++++------ docs/src/man1/cdist.rst | 35 +++++++++++++++++------------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 3b997f63..a841862e 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -34,7 +34,7 @@ dest="$__cdist_abs_mydir/$filename" cd "$__cdist_abs_mydir" exec > "$dest" -cat << eof +cat << eof Reference ========= Variable, path and type reference for cdist @@ -51,7 +51,7 @@ eof done ) -cat << eof +cat << eof Paths ----- @@ -187,13 +187,13 @@ usable within a object directory: files This directory is reserved for user data and will not be used - by cdist at any time. It can be used freely by the type + by cdist at any time. It can be used freely by the type (for instance to store template results). changed This empty file exists in an object directory, if the object has code to be executed (either remote or local). stdin - This file exists and contains data, if data was provided on stdin + This file exists and contains data, if data was provided on stdin when the type was called. @@ -222,65 +222,89 @@ __cdist_log_level, __cdist_log_level_name | TRACE | 5 | +----------------+-----------------+ + Available for: initial manifest, explorer, type manifest, type explorer, + type gencode. +__cdist_colored_log + whether or not cdist's log has colors enabled. + Is set to the string ``true`` if cdist's output is using colors, + otherwise the variable contains the string ``false``. + Available for: initial manifest, explorer, type manifest, type explorer, type gencode. __cdist_dry_run Is set only when doing dry run (``-n`` flag). + Available for: initial manifest, explorer, type manifest, type explorer, type gencode. __explorer Directory that contains all global explorers. + Available for: initial manifest, explorer, type explorer, shell. __files Directory that contains content from the "files" subdirectories from the configuration directories. + Available for: initial manifest, type manifest, type gencode, shell. __manifest Directory that contains the initial manifest. + Available for: initial manifest, type manifest, shell. __global Directory that contains generic output like explorer. + Available for: initial manifest, type manifest, type gencode, shell. __messages_in File to read messages from. + Available for: initial manifest, type manifest, type gencode. __messages_out File to write messages. + Available for: initial manifest, type manifest, type gencode. __object Directory that contains the current object. + Available for: type manifest, type explorer, type gencode and code scripts. __object_id The type unique object id. + Available for: type manifest, type explorer, type gencode and code scripts. - Note: The leading and the trailing "/" will always be stripped (caused by - the filesystem database and ensured by the core). - Note: Double slashes ("//") will not be fixed and result in an error. + + | Note: The leading and the trailing "/" will always be stripped (caused by + the filesystem database and ensured by the core). + | Note: Double slashes ("//") will not be fixed and result in an error. __object_name The full qualified name of the current object. + Available for: type manifest, type explorer, type gencode. __target_host The host we are deploying to. This is primary variable. It's content is literally the one user passed in. + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __target_hostname The hostname of host we are deploying to. This variable is derived from **__target_host** (using **socket.getaddrinfo(__target_host)** and then **socket.gethostbyaddr()**). + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __target_fqdn The fully qualified domain name of the host we are deploying to. This variable is derived from **__target_host** (using **socket.getfqdn()**). + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __target_host_tags Comma separated list of target host tags. + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __type Path to the current type. + Available for: type manifest, type gencode. __type_explorer Directory that contains the type explorers. + Available for: type explorer. Environment variables (for writing) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 4c34c4b7..4def97b5 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -112,9 +112,13 @@ All commands accept the following options: Show the help screen. **--colors COLORED_OUTPUT** - Use a colored output for different log levels.It can - be a boolean or "auto" (default) which enables this - feature if stdout is a tty and disables it otherwise. + Colorize cdist's output. If enabled, cdist will use different colors for + different log levels. + COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off', + 'true'/'false', '1'/'0', and 'auto' (the default). + + If the value is 'auto', colored output is enabled if stdout is a TTY + unless the NO_COLOR (https://no-color.org/) environment variable is defined. **-l LOGLEVEL, --log-level LOGLEVEL** Set the specified verbosity level. The levels, in @@ -168,7 +172,7 @@ Install command is currently in beta. **-b, --beta** Enable beta functionality. - + **-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** Specify custom cache path pattern. If it is not set then default hostdir is used. For more info on format see @@ -191,7 +195,7 @@ Install command is currently in beta. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -306,7 +310,7 @@ Add host(s) to inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -336,7 +340,7 @@ Add tag(s) to inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -379,7 +383,7 @@ Delete host(s) from inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -413,7 +417,7 @@ Delete tag(s) from inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -460,7 +464,7 @@ List inventory database. **-I INVENTORY_DIR, --inventory INVENTORY_DIR** Use specified custom inventory directory. Inventory - directory is set up by the following rules: if cdist + directory is set up by the following rules: if cdist configuration resolves this value then specified directory is used, if HOME env var is set then ~/.cdit/inventory is used, otherwise distribution @@ -685,6 +689,9 @@ The possible keywords and their meanings are as follows: :strong:`cache_path_pattern` Specify cache path pattern. +:strong:`colored_output` + Colorize cdist's output. cf. the :code:`--colors` option. + :strong:`conf_dir` List of configuration directories separated with the character conventionally used by the operating system to separate search path components (as in PATH), @@ -738,7 +745,7 @@ The possible keywords and their meanings are as follows: in the format: YYYYMMDDHHMMSS.us. :strong:`verbosity` - Set verbosity level. Valid values are: + Set verbosity level. Valid values are: 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. @@ -770,7 +777,7 @@ cdist/preos NOTES ----- cdist detects if host is specified by IPv6 address. If so then remote_copy -command is executed with host address enclosed in square brackets +command is executed with host address enclosed in square brackets (see :strong:`scp`\ (1)). EXAMPLES @@ -906,9 +913,7 @@ CDIST_CACHE_PATH_PATTERN Custom cache path pattern. CDIST_COLORED_OUTPUT - Use a colored output for different log levels. - It can be a boolean or 'auto' (default) which enables this feature if - stdout is a tty and disables it otherwise. + Colorize cdist's output. cf. the :code:`--colors` option. CDIST_CONFIG_FILE Custom configuration file. From 7a570f8692a727662a6eda2c13f2b578b55864e4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Fri, 5 Jun 2020 13:59:17 +0200 Subject: [PATCH 13/37] [cdist.cfg.skeleton] Update colored_output documentation based on cdist(1) --- configuration/cdist.cfg.skeleton | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 0730201d..9c66b51b 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -14,11 +14,12 @@ # cache_path_pattern = %h # # colored_output -# Use a colored output for different log levels. -# It can be a boolean or 'auto' (default) which enables this feature if -# stdout is a tty and disables it otherwise. -# Colored output is always disabled if the NO_COLOR environment variable is -# defined (https://no-color.org/). +# Colorize cdist's output. If enabled, cdist will use different colors for +# different log levels. +# Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0', +# and 'auto' +# If the value is 'auto', colored output is enabled if stdout is a TTY +# unless the NO_COLOR (https://no-color.org/) environment variable is defined. # colored_output = auto # # conf_dir From 23e66e08fab14f254c426c88271f461fe8499ba4 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sat, 6 Jun 2020 13:39:29 +0200 Subject: [PATCH 14/37] Restrict colored_output value to always/never/auto. --- cdist/argparse.py | 7 +++---- cdist/configuration.py | 8 +++++--- configuration/cdist.cfg.skeleton | 7 +++---- docs/src/cdist-reference.rst.sh | 6 +++--- docs/src/man1/cdist.rst | 6 +++--- 5 files changed, 17 insertions(+), 17 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 0782654f..85dba246 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -128,10 +128,9 @@ def get_parsers(): parser['colored_output'] = argparse.ArgumentParser(add_help=False) parser['colored_output'].add_argument( - '--colors', - help='Use a colored output for different log levels.' - 'It can be a boolean or "auto" (default) which enables this ' - 'feature if stdout is a tty and disables it otherwise.', + '--colors', metavar='WHEN', + help="Colorize cdist's output based on log level; " + "WHEN is 'always', 'never', or 'auto'.", action='store', dest='colored_output', required=False, choices=cdist.configuration.ColoredOutputOption.CHOICES) diff --git a/cdist/configuration.py b/cdist/configuration.py index a12940b0..4ba43a7a 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -248,7 +248,7 @@ class LogLevelOption(OptionBase): class ColoredOutputOption(BooleanOption): - CHOICES = tuple(configparser.ConfigParser.BOOLEAN_STATES) + ('auto',) + CHOICES = ('always', 'never', 'auto') DEFAULT = 'auto' def get_converter(self): @@ -258,8 +258,10 @@ class ColoredOutputOption(BooleanOption): def translate(val): if isinstance(val, bool): return val - elif val in configparser.ConfigParser.BOOLEAN_STATES: - return configparser.ConfigParser.BOOLEAN_STATES[val] + elif val == 'always': + return True + elif val == 'never': + return False elif val == 'auto': return 'NO_COLOR' not in os.environ and sys.stdout.isatty() diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton index 9c66b51b..14861592 100644 --- a/configuration/cdist.cfg.skeleton +++ b/configuration/cdist.cfg.skeleton @@ -16,10 +16,9 @@ # colored_output # Colorize cdist's output. If enabled, cdist will use different colors for # different log levels. -# Recognized values are 'yes'/'no', 'on'/'off', 'true'/'false', '1'/'0', -# and 'auto' -# If the value is 'auto', colored output is enabled if stdout is a TTY -# unless the NO_COLOR (https://no-color.org/) environment variable is defined. +# Recognized values are 'always', 'never', and 'auto'. +# If the value is 'auto', colors are enabled if stdout is a TTY unless +# the NO_COLOR (https://no-color.org/) environment variable is defined. # colored_output = auto # # conf_dir diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index a841862e..c0ac2c3e 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -369,9 +369,9 @@ CDIST_BETA Enable beta functionalities. CDIST_COLORED_OUTPUT - Use a colored output for different log levels. - It can be a boolean or 'auto' (default) which enables this feature if - stdout is a tty and disables it otherwise. + Colorize cdist's output. If enabled, cdist will use different colors for + different log levels. + Recognized values are 'always', 'never', and 'auto' (the default). CDIST_CACHE_PATH_PATTERN Custom cache path pattern. diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 4def97b5..760ff8f6 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -111,11 +111,11 @@ All commands accept the following options: **-h, --help** Show the help screen. -**--colors COLORED_OUTPUT** +**--colors WHEN** Colorize cdist's output. If enabled, cdist will use different colors for different log levels. - COLORED_OUTPUT recognizes the boolean values 'yes'/'no', 'on'/'off', - 'true'/'false', '1'/'0', and 'auto' (the default). + COLORED_OUTPUT recognizes the values 'always', 'never', + and 'auto' (the default). If the value is 'auto', colored output is enabled if stdout is a TTY unless the NO_COLOR (https://no-color.org/) environment variable is defined. From 89ebd7a4f79bef894cb3f31931571cf577778b26 Mon Sep 17 00:00:00 2001 From: Dennis Camera Date: Sun, 7 Jun 2020 16:45:48 +0200 Subject: [PATCH 15/37] cdist man page: update --colors metavar name --- docs/src/man1/cdist.rst | 62 ++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 35 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 760ff8f6..aae98538 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -15,21 +15,19 @@ SYNOPSIS cdist banner [-h] [-l LOGLEVEL] [-q] [-v] - cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] - [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] - [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] [-4] [-6] - [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] - [-j [JOBS]] [-n] [-o OUT_PATH] [-P] - [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] + cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] + [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] @@ -37,35 +35,31 @@ SYNOPSIS cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... - cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-f HOSTFILE] + cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE] [host [host ...]] - cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] - [-t TAGLIST] - [host [host ...]] - - cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-a] [-f HOSTFILE] - [host [host ...]] - - cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] [host [host ...]] - cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] - [--colors COLORED_OUTPUT] [-g CONFIG_FILE] - [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] + cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] + [host [host ...]] + + cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [host [host ...]] + + cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + [-H] [-t] [host [host ...]] - cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] - [-L] + cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] [-L] [preos] ... cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] @@ -89,8 +83,7 @@ SYNOPSIS [-S SCRIPT] [-s SUITE] [-y REMOTE_COPY] target_dir - cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors COLORED_OUTPUT] - [-s SHELL] + cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors WHEN] [-s SHELL] cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] @@ -114,8 +107,7 @@ All commands accept the following options: **--colors WHEN** Colorize cdist's output. If enabled, cdist will use different colors for different log levels. - COLORED_OUTPUT recognizes the values 'always', 'never', - and 'auto' (the default). + WHEN recognizes the values 'always', 'never', and 'auto' (the default). If the value is 'auto', colored output is enabled if stdout is a TTY unless the NO_COLOR (https://no-color.org/) environment variable is defined. From be47619b1ed7a87090f7615c19a9504f7cf2b590 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Jun 2020 09:11:51 +0200 Subject: [PATCH 16/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 5b57e98b..94ef55d8 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,7 @@ Changelog next: * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) + * Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera) * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) 6.5.6: 2020-05-25 From f5630297bd58fc92d95ab174ed6c8a8e14abf0b9 Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:06:49 +0300 Subject: [PATCH 17/37] [docs] Fixed typo in cdist-configuration.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst index 706ed761..a2024584 100644 --- a/docs/src/cdist-configuration.rst +++ b/docs/src/cdist-configuration.rst @@ -21,7 +21,7 @@ precedence. Configuration option value read from source with higher precedence will overwrite option value, if exists, read from source with lower precedence. That means that command-line option wins them all. -Users can decide on the local conifguration file location. It can be either +Users can decide on the local configuration file location. It can be either ~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist, then ~/.cdist.cfg is used. From 3d725f12dace8d2e977d408b7c01fa28b8b17bde Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:10:27 +0300 Subject: [PATCH 18/37] [docs] Fixed a typo and s/posix/POSIX/ in cdist-why.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-why.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/cdist-why.rst b/docs/src/cdist-why.rst index 1123a1de..0e2cd34d 100644 --- a/docs/src/cdist-why.rst +++ b/docs/src/cdist-why.rst @@ -21,7 +21,7 @@ Not only is shell scripting widely known by system engineers, but it is also a very powerful language. Here are some features which make daily work easy: - * Configuration can react dynamicly on explored values + * Configuration can react dynamically on explored values * High level string manipulation (using sed, awk, grep) * Conditional support (**if, case**) * Loop support (**for, while**) @@ -44,7 +44,7 @@ Cdist requires very little on a target system. Even better, in almost all cases all dependencies are usually fulfilled. Cdist does not require an agent or high level programming languages on the target host: it will run on any host that -has a **ssh server running** and a posix compatible shell +has a **ssh server running** and a POSIX compatible shell (**/bin/sh**). Compared to other configuration management systems, it does not require to open up an additional port. From 46574fc5779b939db5d90e8c11f28e808be458be Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:12:10 +0300 Subject: [PATCH 19/37] [docs] Fixed three typos in cdist-upgrade.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-upgrade.rst | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/cdist-upgrade.rst b/docs/src/cdist-upgrade.rst index 67fd4934..f745b212 100644 --- a/docs/src/cdist-upgrade.rst +++ b/docs/src/cdist-upgrade.rst @@ -14,7 +14,7 @@ To upgrade cdist in the current branch use make man export MANPATH=$MANPATH:$(pwd -P)/doc/man -If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break. +If you stay on a version branch (i.e. 1.0, 1.1., ...), nothing should break. The master branch on the other hand is the development branch and may not be working, break your setup or eat the tree in your garden. @@ -61,7 +61,7 @@ After that, you can go back and continue the upgrade: Update the python package ------------------------- -To upgrade to the lastet version do +To upgrade to the latest version do .. code-block:: sh @@ -158,7 +158,7 @@ Updating from 1.5 to 1.6 * If you used **\_\_package_apt --preseed**, you need to use the new type **\_\_debconf_set_selections** instead. * The **\_\_package** types accepted either --state deinstalled or - --state uninstaaled. Starting with 1.6, it was made consistently + --state uninstalled. Starting with 1.6, it was made consistently to --state removed. Updating from 1.3 to 1.5 From e1ff1bfdffd9a188a045669c4168dcd1fb561b0b Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:15:38 +0300 Subject: [PATCH 20/37] [docs] Fixed two typos in cdist-real-world.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-real-world.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst index ba118d63..973dcd6d 100644 --- a/docs/src/cdist-real-world.rst +++ b/docs/src/cdist-real-world.rst @@ -198,7 +198,7 @@ We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$u Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**. If this directory does not exist then **__file** type would error. We also use stdin as file content source. For details see `Input from stdin `_. -For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent +For feeding stdin we use here-document (**<<** operator). It allows redirection of subsequent lines read by the shell to the input of a command until a line containing only the delimiter and a newline, with no blank characters in between (EOF in our case). @@ -546,7 +546,7 @@ we have changed our **wsgi.py** file uWSGI reloads the application. Our application selects and lists items from **items** table. -Openning application +Opening application ~~~~~~~~~~~~~~~~~~~~ Finally try the application:: From 978e24904336e6af074a7de757282456ff0314a4 Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:23:49 +0300 Subject: [PATCH 21/37] [docs] Fixed capitalization of POSIX in cdist-install.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-install.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 880b9f8e..0b65cad6 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -9,7 +9,7 @@ Source Host This is the machine from which you will configure target hosts. - * /bin/sh: A posix like shell (for instance bash, dash, zsh) + * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * Python >= 3.2 * SSH client * sphinx (for building html docs and/or the man pages) @@ -17,7 +17,7 @@ This is the machine from which you will configure target hosts. Target Hosts ~~~~~~~~~~~~ - * /bin/sh: A posix like shell (for instance bash, dash, zsh) + * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) * SSH server Install cdist From dc018fdb16ba8b6044bbb55efd44e975fc5247ae Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:25:46 +0300 Subject: [PATCH 22/37] [docs] Fixed typo in cdist-cache.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-cache.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst index 0e5361ee..d2d2d56c 100644 --- a/docs/src/cdist-cache.rst +++ b/docs/src/cdist-cache.rst @@ -59,7 +59,7 @@ typeorder Object cache overview ~~~~~~~~~~~~~~~~~~~~~ -Each object under :strong:`object` directory has its own structurue. +Each object under :strong:`object` directory has its own structure. code-local code generated from gencode-local, present only if something is From 7b262c0cec2c3c42822cef103220965424500bcb Mon Sep 17 00:00:00 2001 From: Jaak Ristioja Date: Wed, 3 Jun 2020 23:27:16 +0300 Subject: [PATCH 23/37] [docs] Fixed capitalization of URLs in cdist-bootstrap.rst Signed-off-by: Jaak Ristioja --- docs/src/cdist-bootstrap.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/cdist-bootstrap.rst b/docs/src/cdist-bootstrap.rst index c9972a99..10f86c5a 100644 --- a/docs/src/cdist-bootstrap.rst +++ b/docs/src/cdist-bootstrap.rst @@ -25,7 +25,7 @@ people, have a look at `cdist best practice `_. Setup working directory and branch ---------------------------------- I assume you have a fresh copy of the cdist tree in ~/cdist, cloned from -one of the official urls (see `cdist quickstart `_ if you don't). +one of the official URLs (see `cdist quickstart `_ if you don't). Entering the command "git branch" should show you "* master", which indicates you are on the **master** branch. From 191f45eb7f8ab8ce97f996f3bb8375da5142d39a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Jun 2020 13:48:37 +0200 Subject: [PATCH 24/37] ++changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 94ef55d8..e73cfa6f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -6,6 +6,8 @@ next: * Core: Support colored log output (Evil Ham) * Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera) * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) + * Documentation: Fix trivial grammatical mistakes (Jaak Ristioja) + * Explorer os: Fix for sles15 (Daniel Heule) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 955243a93b28d3524f93eb6d69af1f0eec5ff36d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 9 Jun 2020 12:51:19 +0200 Subject: [PATCH 25/37] Update cdist man page copyright years --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index aae98538..68a9a4ee 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -969,5 +969,5 @@ such case and display a warning message. An example of such a case: COPYING ------- -Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is +Copyright \(C) 2011-2020 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License v3 or later (GPLv3+). From 7c490a703d0203747e481a9fc3405bf7eb5ba01a Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Wed, 10 Jun 2020 11:38:14 +0300 Subject: [PATCH 26/37] [__clean_path] add --path parameter --- cdist/conf/type/__clean_path/explorer/list | 7 ++++++- cdist/conf/type/__clean_path/gencode-remote | 7 ++++++- cdist/conf/type/__clean_path/man.rst | 10 +++++++++- cdist/conf/type/__clean_path/parameter/optional | 1 + 4 files changed, 22 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list index 07d38127..2bdc63a5 100755 --- a/cdist/conf/type/__clean_path/explorer/list +++ b/cdist/conf/type/__clean_path/explorer/list @@ -18,7 +18,12 @@ # along with cdist. If not, see . # -path="/$__object_id" +if [ -f "$__object/parameter/path" ] +then + path="$( cat "$__object/parameter/path" )" +else + path="/$__object_id" +fi [ ! -d "$path" ] && exit 0 diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote index 998a70d8..2899c4a5 100755 --- a/cdist/conf/type/__clean_path/gencode-remote +++ b/cdist/conf/type/__clean_path/gencode-remote @@ -20,7 +20,12 @@ [ ! -s "$__object/explorer/list" ] && exit 0 -path="/$__object_id" +if [ -f "$__object/parameter/path" ] +then + path="$( cat "$__object/parameter/path" )" +else + path="/$__object_id" +fi pattern="$( cat "$__object/parameter/pattern" )" diff --git a/cdist/conf/type/__clean_path/man.rst b/cdist/conf/type/__clean_path/man.rst index 826f4589..31d90701 100644 --- a/cdist/conf/type/__clean_path/man.rst +++ b/cdist/conf/type/__clean_path/man.rst @@ -10,7 +10,7 @@ DESCRIPTION ----------- Remove files and directories which match the pattern. -Provided path (as __object_id) must be a directory. +Provided path must be a directory. Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details. @@ -29,6 +29,9 @@ pattern OPTIONAL PARAMETERS ------------------- +path + Path which will be cleaned. Defaults to ``$__object_id``. + exclude Pattern of files which are excluded from removal. @@ -46,6 +49,11 @@ EXAMPLES --exclude '.+\(charset\.conf\|security\.conf\)' \ --onchange 'service apache2 restart' + __clean_path apache2-conf-enabled \ + --path /etc/apache2/conf-enabled \ + --pattern '.+' \ + --exclude '.+\(charset\.conf\|security\.conf\)' \ + --onchange 'service apache2 restart' AUTHORS ------- diff --git a/cdist/conf/type/__clean_path/parameter/optional b/cdist/conf/type/__clean_path/parameter/optional index 6f313474..3b97f71c 100644 --- a/cdist/conf/type/__clean_path/parameter/optional +++ b/cdist/conf/type/__clean_path/parameter/optional @@ -1,2 +1,3 @@ exclude onchange +path From 74e5d7182a466cd02b79137f1d73500c0cc50808 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Jun 2020 10:45:20 +0200 Subject: [PATCH 27/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index e73cfa6f..bcde9268 100644 --- a/docs/changelog +++ b/docs/changelog @@ -8,6 +8,7 @@ next: * Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak) * Documentation: Fix trivial grammatical mistakes (Jaak Ristioja) * Explorer os: Fix for sles15 (Daniel Heule) + * Type __clean_path: Add --path parameter (Ander Punnar) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 840e417eb79c7135762d3353f34a3f8330cb0e84 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 11 Jun 2020 14:16:37 +0200 Subject: [PATCH 28/37] Fix emulator colored logging --- cdist/argparse.py | 2 +- cdist/core/manifest.py | 3 ++- cdist/emulator.py | 2 +- cdist/flock.py | 1 + cdist/log.py | 16 +++++++--------- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 85dba246..1c83237c 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -498,7 +498,7 @@ def handle_loglevel(args): def handle_log_colors(args): if cdist.configuration.ColoredOutputOption.translate(args.colored_output): - cdist.log.DefaultLog.USE_COLORS = True + cdist.log.CdistFormatter.USE_COLORS = True def parse_and_configure(argv, singleton=True): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8b833ff2..2e7b873e 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -119,7 +119,8 @@ class Manifest(object): '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( self.log), - '__cdist_colored_log': str(self.log.USE_COLORS).lower(), + '__cdist_colored_log': str( + cdist.log.CdistFormatter.USE_COLORS).lower(), } if dry_run: diff --git a/cdist/emulator.py b/cdist/emulator.py index 4eaf2c93..08e15a78 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -130,7 +130,7 @@ class Emulator(object): logging.root.setLevel(logging.WARNING) colored_log = self.env.get('__cdist_colored_log', 'false') - cdist.log.ColorFormatter.USE_COLORS = colored_log == 'true' + cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true' self.log = logging.getLogger(self.target_host[0]) diff --git a/cdist/flock.py b/cdist/flock.py index d8bac916..3afacfd9 100644 --- a/cdist/flock.py +++ b/cdist/flock.py @@ -22,6 +22,7 @@ import fcntl import logging import os +import cdist.log log = logging.getLogger('cdist-flock') diff --git a/cdist/log.py b/cdist/log.py index 19efebdb..efae2cd3 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -50,7 +50,8 @@ def _trace(msg, *args, **kwargs): logging.trace = _trace -class ColorFormatter(logging.Formatter): +class CdistFormatter(logging.Formatter): + USE_COLORS = False RESET = '\033[0m' COLOR_MAP = { 'ERROR': '\033[0;31m', @@ -66,14 +67,14 @@ class ColorFormatter(logging.Formatter): def format(self, record): msg = super().format(record) - color = self.COLOR_MAP.get(record.levelname) - if color: - msg = color + msg + self.RESET + if self.USE_COLORS: + color = self.COLOR_MAP.get(record.levelname) + if color: + msg = color + msg + self.RESET return msg class DefaultLog(logging.Logger): - USE_COLORS = False FORMAT = '%(levelname)s: %(message)s' class StdoutFilter(logging.Filter): @@ -88,10 +89,7 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - if self.USE_COLORS: - formatter = ColorFormatter(self.FORMAT) - else: - formatter = logging.Formatter(self.FORMAT) + formatter = CdistFormatter(self.FORMAT) self.addFilter(self) From 4167f9f60c30a4587b492c2fa3dce2f02ded463a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 11 Jun 2020 14:22:54 +0200 Subject: [PATCH 29/37] Use proper format string with name --- cdist/log.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index efae2cd3..790059df 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -75,7 +75,7 @@ class CdistFormatter(logging.Formatter): class DefaultLog(logging.Logger): - FORMAT = '%(levelname)s: %(message)s' + FORMAT = '%(levelname)s: %(name)s: %(message)s' class StdoutFilter(logging.Filter): def filter(self, rec): @@ -91,8 +91,6 @@ class DefaultLog(logging.Logger): formatter = CdistFormatter(self.FORMAT) - self.addFilter(self) - stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.addFilter(self.StdoutFilter()) stdout_handler.setLevel(logging.TRACE) @@ -106,13 +104,6 @@ class DefaultLog(logging.Logger): self.addHandler(stdout_handler) self.addHandler(stderr_handler) - def filter(self, record): - """Prefix messages with logger name""" - - record.msg = self.name + ": " + str(record.msg) - - return True - def verbose(self, msg, *args, **kwargs): self.log(logging.VERBOSE, msg, *args, **kwargs) @@ -134,7 +125,7 @@ class TimestampingLog(DefaultLog): class ParallelLog(DefaultLog): - FORMAT = '%(levelname)s: [%(process)d]: %(message)s' + FORMAT = '%(levelname)s: [%(process)d]: %(name)s: %(message)s' class TimestampingParallelLog(TimestampingLog, ParallelLog): From eec7ab8e4577008c6a48a473e295a9d1e31f2358 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Jun 2020 16:59:18 +0200 Subject: [PATCH 30/37] Increase minimum supported Python version to 3.5 --- docs/src/cdist-install.rst | 2 +- docs/src/man1/cdist.rst | 2 +- scripts/cdist | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 0b65cad6..6f4f14d7 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -10,7 +10,7 @@ Source Host This is the machine from which you will configure target hosts. * /bin/sh: A POSIX like shell (for instance bash, dash, zsh) - * Python >= 3.2 + * Python >= 3.5 * SSH client * sphinx (for building html docs and/or the man pages) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 68a9a4ee..9bf8fc9b 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -95,7 +95,7 @@ cdist is the frontend executable to the cdist configuration management. It supports different subcommands as explained below. It is written in Python so it requires :strong:`python`\ (1) to be installed. -It requires a minimal Python version 3.2. +It requires a minimal Python version 3.5. GENERAL ------- diff --git a/scripts/cdist b/scripts/cdist index 7bf12c01..664504a0 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -60,7 +60,7 @@ def commandline(): if __name__ == "__main__": - cdistpythonversion = '3.2' + cdistpythonversion = '3.5' if sys.version < cdistpythonversion: print('Python >= {} is required on the source host.'.format( cdistpythonversion), file=sys.stderr) From eba3d0505bba4ee0472df3452142cc63c8fe2f5c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 12 Jun 2020 06:29:39 +0200 Subject: [PATCH 31/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index bcde9268..0cc2a714 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ next: * Documentation: Fix trivial grammatical mistakes (Jaak Ristioja) * Explorer os: Fix for sles15 (Daniel Heule) * Type __clean_path: Add --path parameter (Ander Punnar) + * Core: Increase minimal supported Python version to 3.5 (Darko Poljak) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From 6e9e9ad557479354938257f04c8fab283b8251e8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 12 Nov 2019 00:40:58 +0100 Subject: [PATCH 32/37] implement log server to capture nested logging output Signed-off-by: Steven Armstrong --- cdist/core/code.py | 4 +++ cdist/install.py | 23 +++++++++++++- cdist/log.py | 77 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 90 insertions(+), 14 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 1550880a..a7d9b7ca 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -116,6 +116,10 @@ class Code(object): if dry_run: self.env['__cdist_dry_run'] = '1' + if '__cdist_log_server_socket_to_export' in os.environ: + self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export'] + + def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type script = os.path.join(self.local.type_path, diff --git a/cdist/install.py b/cdist/install.py index b88ad016..3f94ca68 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2013-2019 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -20,11 +20,32 @@ # # +import os +import logging +import tempfile + import cdist.config import cdist.core class Install(cdist.config.Config): + + @classmethod + def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, + parallel, configuration, remove_remote_files_dirs=False): + # Start a log server so nested `cdist config` runs have a place to + # send their logs to. + log_server_socket_dir = tempfile.mkdtemp() + log_server_socket = os.path.join(log_server_socket_dir, 'log-server') + cls._register_path_for_removal(log_server_socket_dir) + log = logging.getLogger(host) + log.debug('Starting logging server on: %s', log_server_socket) + os.environ['__cdist_log_server_socket_to_export'] = log_server_socket + cdist.log.setupLogServer(log_server_socket) + + super().onehost(host, host_tags, host_base_path, host_dir_name, args, + parallel, configuration, remove_remote_files_dirs=False) + def object_list(self): """Short name for object list retrieval. In install mode, we only care about install objects. diff --git a/cdist/log.py b/cdist/log.py index 790059df..94dd11e8 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2019-2020 Steven Armstrong # # This file is part of cdist. # @@ -20,9 +21,17 @@ # # -import logging -import sys +import asyncio +import contextlib import datetime +import logging +import logging.handlers +import os +import pickle +import struct +import sys +import threading +import time # Define additional cdist logging levels. @@ -89,20 +98,25 @@ class DefaultLog(logging.Logger): super().__init__(name) self.propagate = False - formatter = CdistFormatter(self.FORMAT) + if '__cdist_log_server_socket' in os.environ: + log_server_socket = os.environ['__cdist_log_server_socket'] + socket_handler = logging.handlers.SocketHandler(log_server_socket, None) + self.addHandler(socket_handler) + else: + formatter = CdistFormatter(self.FORMAT) - stdout_handler = logging.StreamHandler(sys.stdout) - stdout_handler.addFilter(self.StdoutFilter()) - stdout_handler.setLevel(logging.TRACE) - stdout_handler.setFormatter(formatter) + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.addFilter(self.StdoutFilter()) + stdout_handler.setLevel(logging.TRACE) + stdout_handler.setFormatter(formatter) - stderr_handler = logging.StreamHandler(sys.stderr) - stderr_handler.addFilter(self.StderrFilter()) - stderr_handler.setLevel(logging.ERROR) - stderr_handler.setFormatter(formatter) + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.addFilter(self.StderrFilter()) + stderr_handler.setLevel(logging.ERROR) + stderr_handler.setFormatter(formatter) - self.addHandler(stdout_handler) - self.addHandler(stderr_handler) + self.addHandler(stdout_handler) + self.addHandler(stderr_handler) def verbose(self, msg, *args, **kwargs): self.log(logging.VERBOSE, msg, *args, **kwargs) @@ -152,4 +166,41 @@ def setupParallelLogging(): logging.setLoggerClass(ParallelLog) +async def handle_log_client(reader, writer): + while True: + chunk = await reader.read(4) + if len(chunk) < 4: + return + + data_size = struct.unpack('>L', chunk)[0] + data = bytearray(data_size) + view = memoryview(data) + data_pending = data_size + data = await reader.read(data_size) + + obj = pickle.loads(data) + record = logging.makeLogRecord(obj) + logger = logging.getLogger(record.name) + logger.handle(record) + + +def run_log_server(server_address): + # Get a new loop inside the current thread to run the log server. + loop = asyncio.new_event_loop() + loop.create_task(asyncio.start_unix_server(handle_log_client, server_address)) + loop.run_forever() + + +def setupLogServer(log_server_socket): + """Run a asyncio based unix socket log server in a background thread. + """ + with contextlib.suppress(FileNotFoundError): + os.remove(log_server_socket) + t = threading.Thread(target=run_log_server, args=(log_server_socket,)) + # Deamonizing the thread means we don't have to care about stoping it. + # It will die together with the main process. + t.daemon = True + t.start() + + setupDefaultLogging() From 831bfc822b61a5e5cf8186a32ea592622693efdb Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 12 Nov 2019 09:22:28 +0100 Subject: [PATCH 33/37] remove unused code Signed-off-by: Steven Armstrong --- cdist/log.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/log.py b/cdist/log.py index 94dd11e8..c1376a58 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -173,9 +173,6 @@ async def handle_log_client(reader, writer): return data_size = struct.unpack('>L', chunk)[0] - data = bytearray(data_size) - view = memoryview(data) - data_pending = data_size data = await reader.read(data_size) obj = pickle.loads(data) From 57e352cd1e7b400ca1daf7e1e680874bd9a6cc01 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 11 Jan 2020 01:59:18 +0100 Subject: [PATCH 34/37] log server is also usefull for cdist config Signed-off-by: Steven Armstrong --- cdist/argparse.py | 6 ++++++ cdist/config.py | 10 ++++++++++ cdist/install.py | 13 +++---------- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 1c83237c..15b9ca7a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -250,6 +250,12 @@ def get_parsers(): '-S', '--disable-saving-output-streams', help='Disable saving output streams.', action='store_false', dest='save_output_streams', default=True) + parser['config_main'].add_argument( + '--log-server', + action='store_true', + help=('Start a log server for sub processes to use.' + 'This is mainly usefull when running cdist nested' + 'from a code-local script.')) # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) diff --git a/cdist/config.py b/cdist/config.py index b2d72f05..82e8559b 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -385,6 +385,16 @@ class Config(object): log = logging.getLogger(host) + if args.log_server: + # Start a log server so that nested `cdist config` runs have a place + # to send their logs to. + log_server_socket_dir = tempfile.mkdtemp() + log_server_socket = os.path.join(log_server_socket_dir, 'log-server') + cls._register_path_for_removal(log_server_socket_dir) + log.debug('Starting logging server on: %s', log_server_socket) + os.environ['__cdist_log_server_socket_to_export'] = log_server_socket + cdist.log.setupLogServer(log_server_socket) + try: remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) diff --git a/cdist/install.py b/cdist/install.py index 3f94ca68..ec0a7270 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -33,16 +33,9 @@ class Install(cdist.config.Config): @classmethod def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, parallel, configuration, remove_remote_files_dirs=False): - # Start a log server so nested `cdist config` runs have a place to - # send their logs to. - log_server_socket_dir = tempfile.mkdtemp() - log_server_socket = os.path.join(log_server_socket_dir, 'log-server') - cls._register_path_for_removal(log_server_socket_dir) - log = logging.getLogger(host) - log.debug('Starting logging server on: %s', log_server_socket) - os.environ['__cdist_log_server_socket_to_export'] = log_server_socket - cdist.log.setupLogServer(log_server_socket) - + # Always start log server during cdist install so that nested + # `cdist config` runs have a place to send their logs to. + args.log_server = True super().onehost(host, host_tags, host_base_path, host_dir_name, args, parallel, configuration, remove_remote_files_dirs=False) From 59b98091d78c5195d2aa4a2ea3ef79f93e5ba67c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 9 Jun 2020 12:47:50 +0200 Subject: [PATCH 35/37] Adapt; update docs and code style --- cdist/__init__.py | 4 ++++ cdist/argparse.py | 13 +++++++------ cdist/config.py | 27 ++++++++++++--------------- cdist/core/code.py | 6 +++--- cdist/install.py | 8 +++----- cdist/log.py | 18 +++++++++++------- docs/src/man1/cdist.rst | 26 ++++++++++++++++---------- scripts/cdist | 5 ++--- 8 files changed, 58 insertions(+), 49 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index c673b3ba..be573170 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -26,6 +26,7 @@ import hashlib import cdist.log import cdist.version + VERSION = cdist.version.VERSION BANNER = """ @@ -48,6 +49,9 @@ REMOTE_EXEC = "ssh -o User=root" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" +MIN_SUPPORTED_PYTHON_VERSION = '3.5' + + class Error(Exception): """Base exception class for this project""" pass diff --git a/cdist/argparse.py b/cdist/argparse.py index 15b9ca7a..77303591 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -206,6 +206,13 @@ def get_parsers(): 'supported. Without argument CPU count is used by default. '), action='store', dest='jobs', const=multiprocessing.cpu_count()) + parser['config_main'].add_argument( + '--log-server', + action='store_true', + help=('Start a log server for sub processes to use. ' + 'This is mainly useful when running cdist nested ' + 'from a code-local script. Log server is alwasy ' + 'implicitly started for \'install\' command.')) parser['config_main'].add_argument( '-n', '--dry-run', help='Do not execute code.', action='store_true') @@ -250,12 +257,6 @@ def get_parsers(): '-S', '--disable-saving-output-streams', help='Disable saving output streams.', action='store_false', dest='save_output_streams', default=True) - parser['config_main'].add_argument( - '--log-server', - action='store_true', - help=('Start a log server for sub processes to use.' - 'This is mainly usefull when running cdist nested' - 'from a code-local script.')) # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) diff --git a/cdist/config.py b/cdist/config.py index 82e8559b..9fe9b676 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -29,18 +29,20 @@ import time import itertools import tempfile import multiprocessing -from cdist.mputil import mp_pool_run, mp_sig_handler import atexit import shutil import socket + +from cdist.mputil import mp_pool_run, mp_sig_handler +from cdist import core, inventory +from cdist.util.remoteutil import inspect_ssh_mux_opts + import cdist import cdist.hostsource import cdist.exec.local import cdist.exec.remote import cdist.util.ipaddr as ipaddr import cdist.configuration -from cdist import core, inventory -from cdist.util.remoteutil import inspect_ssh_mux_opts def graph_check_cycle(graph): @@ -195,7 +197,6 @@ class Config(object): @classmethod def commandline(cls, args): """Configure remote system""" - if (args.parallel and args.parallel != 1) or args.jobs: if args.timestamp: cdist.log.setupTimestampingParallelLogging() @@ -382,20 +383,16 @@ class Config(object): If operating in parallel then return tuple (host, True|False, ) so that main process knows for which host function was successful. """ - log = logging.getLogger(host) - if args.log_server: - # Start a log server so that nested `cdist config` runs have a place - # to send their logs to. - log_server_socket_dir = tempfile.mkdtemp() - log_server_socket = os.path.join(log_server_socket_dir, 'log-server') - cls._register_path_for_removal(log_server_socket_dir) - log.debug('Starting logging server on: %s', log_server_socket) - os.environ['__cdist_log_server_socket_to_export'] = log_server_socket - cdist.log.setupLogServer(log_server_socket) - try: + if args.log_server: + # Start a log server so that nested `cdist config` runs + # have a place to send their logs to. + log_server_socket_dir = tempfile.mkdtemp() + cls._register_path_for_removal(log_server_socket_dir) + cdist.log.setupLogServer(log_server_socket_dir, log) + remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( args) log.debug("remote_exec for host \"{}\": {}".format( diff --git a/cdist/core/code.py b/cdist/core/code.py index a7d9b7ca..2a30908e 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -116,9 +116,9 @@ class Code(object): if dry_run: self.env['__cdist_dry_run'] = '1' - if '__cdist_log_server_socket_to_export' in os.environ: - self.env['__cdist_log_server_socket'] = os.environ['__cdist_log_server_socket_to_export'] - + if '__cdist_log_server_socket_export' in os.environ: + self.env['__cdist_log_server_socket'] = os.environ[ + '__cdist_log_server_socket_export'] def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type diff --git a/cdist/install.py b/cdist/install.py index ec0a7270..561b2fa6 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -20,12 +20,9 @@ # # -import os -import logging -import tempfile - import cdist.config import cdist.core +import cdist.log class Install(cdist.config.Config): @@ -36,8 +33,9 @@ class Install(cdist.config.Config): # Always start log server during cdist install so that nested # `cdist config` runs have a place to send their logs to. args.log_server = True + super().onehost(host, host_tags, host_base_path, host_dir_name, args, - parallel, configuration, remove_remote_files_dirs=False) + parallel, configuration, remove_remote_files_dirs) def object_list(self): """Short name for object list retrieval. diff --git a/cdist/log.py b/cdist/log.py index c1376a58..bee99fac 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -21,17 +21,16 @@ # # -import asyncio -import contextlib import datetime import logging import logging.handlers +import sys import os +import asyncio +import contextlib import pickle import struct -import sys import threading -import time # Define additional cdist logging levels. @@ -100,7 +99,8 @@ class DefaultLog(logging.Logger): if '__cdist_log_server_socket' in os.environ: log_server_socket = os.environ['__cdist_log_server_socket'] - socket_handler = logging.handlers.SocketHandler(log_server_socket, None) + socket_handler = logging.handlers.SocketHandler(log_server_socket, + None) self.addHandler(socket_handler) else: formatter = CdistFormatter(self.FORMAT) @@ -184,13 +184,17 @@ async def handle_log_client(reader, writer): def run_log_server(server_address): # Get a new loop inside the current thread to run the log server. loop = asyncio.new_event_loop() - loop.create_task(asyncio.start_unix_server(handle_log_client, server_address)) + loop.create_task(asyncio.start_unix_server(handle_log_client, + server_address)) loop.run_forever() -def setupLogServer(log_server_socket): +def setupLogServer(socket_dir, log=logging.getLogger(__name__)): """Run a asyncio based unix socket log server in a background thread. """ + log_server_socket = os.path.join(socket_dir, 'log-server') + log.debug('Starting logging server on: %s', log_server_socket) + os.environ['__cdist_log_server_socket_export'] = log_server_socket with contextlib.suppress(FileNotFoundError): os.remove(log_server_socket) t = threading.Thread(target=run_log_server, args=(log_server_socket,)) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 9bf8fc9b..aa2607f8 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -17,20 +17,20 @@ SYNOPSIS cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] - [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-P] - [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] - [-p [HOST_MAX]] [-s] [-t] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server] + [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] + [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN] - [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] - [-P] [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] - [-p [HOST_MAX]] [-s] [-t] + [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server] + [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A] + [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... @@ -202,6 +202,12 @@ Install command is currently in beta. are supported. Without argument CPU count is used by default. +**--log-server** + Start a log server for sub processes to use. This is + mainly useful when running cdist nested from a code- + local script. Log server is always implicitly started + for 'install' command. + **-n, --dry-run** Do not execute code. diff --git a/scripts/cdist b/scripts/cdist index 664504a0..b1d782ab 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -60,10 +60,9 @@ def commandline(): if __name__ == "__main__": - cdistpythonversion = '3.5' - if sys.version < cdistpythonversion: + if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION: print('Python >= {} is required on the source host.'.format( - cdistpythonversion), file=sys.stderr) + cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr) sys.exit(1) exit_code = 0 From 5be8437a602907009d7603fa41ae0a4e6528ffdf Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 13 Jun 2020 13:48:34 +0200 Subject: [PATCH 36/37] ++changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 0cc2a714..fa5f7405 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,7 @@ next: * Explorer os: Fix for sles15 (Daniel Heule) * Type __clean_path: Add --path parameter (Ander Punnar) * Core: Increase minimal supported Python version to 3.5 (Darko Poljak) + * Core: Add log server for nested logging (Steven Armstrong) 6.5.6: 2020-05-25 * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) From cdb998398d25f1ba99f01364bbc42b900be944d5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 17 Jun 2020 12:10:58 +0200 Subject: [PATCH 37/37] Release 6.6.0 --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index fa5f7405..c8388b93 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +6.6.0: 2020-06-17 * Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar) * Core: Support colored log output (Evil Ham) * Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)