Merge branch 'master' into beta

This commit is contained in:
Darko Poljak 2020-06-17 12:19:32 +02:00
commit bf6ce46c95
33 changed files with 434 additions and 111 deletions

View file

@ -26,6 +26,7 @@ import hashlib
import cdist.log import cdist.log
import cdist.version import cdist.version
VERSION = cdist.version.VERSION VERSION = cdist.version.VERSION
BANNER = """ BANNER = """
@ -51,6 +52,9 @@ ORDER_DEP_STATE_NAME = 'order_dep_state'
TYPEORDER_DEP_NAME = 'typeorder_dep' TYPEORDER_DEP_NAME = 'typeorder_dep'
MIN_SUPPORTED_PYTHON_VERSION = '3.5'
class Error(Exception): class Error(Exception):
"""Base exception class for this project""" """Base exception class for this project"""
pass pass

View file

@ -6,6 +6,7 @@ import collections
import functools import functools
import cdist.configuration import cdist.configuration
import cdist.trigger import cdist.trigger
import cdist.log
import cdist.preos import cdist.preos
import cdist.info import cdist.info
@ -126,6 +127,14 @@ def get_parsers():
'value.'), 'value.'),
action='count', default=None) action='count', default=None)
parser['colored_output'] = argparse.ArgumentParser(add_help=False)
parser['colored_output'].add_argument(
'--colors', metavar='WHEN',
help="Colorize cdist's output based on log level; "
"WHEN is 'always', 'never', or 'auto'.",
action='store', dest='colored_output', required=False,
choices=cdist.configuration.ColoredOutputOption.CHOICES)
parser['beta'] = argparse.ArgumentParser(add_help=False) parser['beta'] = argparse.ArgumentParser(add_help=False)
parser['beta'].add_argument( parser['beta'].add_argument(
'-b', '--beta', '-b', '--beta',
@ -198,6 +207,13 @@ def get_parsers():
'supported. Without argument CPU count is used by default. '), 'supported. Without argument CPU count is used by default. '),
action='store', dest='jobs', action='store', dest='jobs',
const=multiprocessing.cpu_count()) const=multiprocessing.cpu_count())
parser['config_main'].add_argument(
'--log-server',
action='store_true',
help=('Start a log server for sub processes to use. '
'This is mainly useful when running cdist nested '
'from a code-local script. Log server is alwasy '
'implicitly started for \'install\' command.'))
parser['config_main'].add_argument( parser['config_main'].add_argument(
'-n', '--dry-run', '-n', '--dry-run',
help='Do not execute code.', action='store_true') help='Do not execute code.', action='store_true')
@ -284,6 +300,7 @@ def get_parsers():
'host', nargs='*', help='Host(s) to operate on.') 'host', nargs='*', help='Host(s) to operate on.')
parser['config'] = parser['sub'].add_parser( parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'], 'config', parents=[parser['loglevel'], parser['beta'],
parser['colored_output'],
parser['common'], parser['common'],
parser['config_main'], parser['config_main'],
parser['inventory_common'], parser['inventory_common'],
@ -302,6 +319,7 @@ def get_parsers():
parser['add-host'] = parser['invsub'].add_parser( parser['add-host'] = parser['invsub'].add_parser(
'add-host', parents=[parser['loglevel'], parser['beta'], 'add-host', parents=[parser['loglevel'], parser['beta'],
parser['colored_output'],
parser['common'], parser['common'],
parser['inventory_common']]) parser['inventory_common']])
parser['add-host'].add_argument( parser['add-host'].add_argument(
@ -316,6 +334,7 @@ def get_parsers():
parser['add-tag'] = parser['invsub'].add_parser( parser['add-tag'] = parser['invsub'].add_parser(
'add-tag', parents=[parser['loglevel'], parser['beta'], 'add-tag', parents=[parser['loglevel'], parser['beta'],
parser['colored_output'],
parser['common'], parser['common'],
parser['inventory_common']]) parser['inventory_common']])
parser['add-tag'].add_argument( parser['add-tag'].add_argument(
@ -347,6 +366,7 @@ def get_parsers():
parser['del-host'] = parser['invsub'].add_parser( parser['del-host'] = parser['invsub'].add_parser(
'del-host', parents=[parser['loglevel'], parser['beta'], 'del-host', parents=[parser['loglevel'], parser['beta'],
parser['colored_output'],
parser['common'], parser['common'],
parser['inventory_common']]) parser['inventory_common']])
parser['del-host'].add_argument( parser['del-host'].add_argument(
@ -364,6 +384,7 @@ def get_parsers():
parser['del-tag'] = parser['invsub'].add_parser( parser['del-tag'] = parser['invsub'].add_parser(
'del-tag', parents=[parser['loglevel'], parser['beta'], 'del-tag', parents=[parser['loglevel'], parser['beta'],
parser['colored_output'],
parser['common'], parser['common'],
parser['inventory_common']]) parser['inventory_common']])
parser['del-tag'].add_argument( parser['del-tag'].add_argument(
@ -399,6 +420,7 @@ def get_parsers():
parser['list'] = parser['invsub'].add_parser( parser['list'] = parser['invsub'].add_parser(
'list', parents=[parser['loglevel'], parser['beta'], 'list', parents=[parser['loglevel'], parser['beta'],
parser['colored_output'],
parser['common'], parser['common'],
parser['inventory_common']]) parser['inventory_common']])
parser['list'].add_argument( parser['list'].add_argument(
@ -431,7 +453,7 @@ def get_parsers():
# Shell # Shell
parser['shell'] = parser['sub'].add_parser( parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']]) 'shell', parents=[parser['loglevel'], parser['colored_output']])
parser['shell'].add_argument( parser['shell'].add_argument(
'-s', '--shell', '-s', '--shell',
help=('Select shell to use, defaults to current shell. Used shell' help=('Select shell to use, defaults to current shell. Used shell'
@ -501,7 +523,12 @@ def handle_loglevel(args):
if hasattr(args, 'quiet') and args.quiet: if hasattr(args, 'quiet') and args.quiet:
args.verbose = _verbosity_level_off args.verbose = _verbosity_level_off
logging.root.setLevel(_verbosity_level[args.verbose]) logging.getLogger().setLevel(_verbosity_level[args.verbose])
def handle_log_colors(args):
if cdist.configuration.ColoredOutputOption.translate(args.colored_output):
cdist.log.CdistFormatter.USE_COLORS = True
def parse_and_configure(argv, singleton=True): def parse_and_configure(argv, singleton=True):
@ -515,6 +542,7 @@ def parse_and_configure(argv, singleton=True):
raise cdist.Error(str(e)) raise cdist.Error(str(e))
# Loglevels are handled globally in here # Loglevels are handled globally in here
handle_loglevel(args) handle_loglevel(args)
handle_log_colors(args)
log = logging.getLogger("cdist") log = logging.getLogger("cdist")

View file

@ -143,6 +143,11 @@ case "$uname_s" in
esac esac
if [ -f /etc/os-release ]; then if [ -f /etc/os-release ]; then
# after sles15, suse don't provide an /etc/SuSE-release anymore, but there is almost no difference between sles and opensuse leap, so call it suse
if grep -q ^ID_LIKE=\"suse\" /etc/os-release 2>/dev/null; then
echo suse
exit 0
fi
# already lowercase, according to: # already lowercase, according to:
# https://www.freedesktop.org/software/systemd/man/os-release.html # https://www.freedesktop.org/software/systemd/man/os-release.html
awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release awk -F= '/^ID=/ { if ($2 ~ /^'"'"'(.*)'"'"'$/ || $2 ~ /^"(.*)"$/) { print substr($2, 2, length($2) - 2) } else { print $2 } }' /etc/os-release

View file

@ -18,7 +18,12 @@
# along with cdist. If not, see <http://www.gnu.org/licenses/>. # along with cdist. If not, see <http://www.gnu.org/licenses/>.
# #
if [ -f "$__object/parameter/path" ]
then
path="$( cat "$__object/parameter/path" )"
else
path="/$__object_id" path="/$__object_id"
fi
[ ! -d "$path" ] && exit 0 [ ! -d "$path" ] && exit 0

View file

@ -20,7 +20,12 @@
[ ! -s "$__object/explorer/list" ] && exit 0 [ ! -s "$__object/explorer/list" ] && exit 0
if [ -f "$__object/parameter/path" ]
then
path="$( cat "$__object/parameter/path" )"
else
path="/$__object_id" path="/$__object_id"
fi
pattern="$( cat "$__object/parameter/pattern" )" pattern="$( cat "$__object/parameter/pattern" )"

View file

@ -10,7 +10,7 @@ DESCRIPTION
----------- -----------
Remove files and directories which match the pattern. Remove files and directories which match the pattern.
Provided path (as __object_id) must be a directory. Provided path must be a directory.
Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details. Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details.
@ -29,6 +29,9 @@ pattern
OPTIONAL PARAMETERS OPTIONAL PARAMETERS
------------------- -------------------
path
Path which will be cleaned. Defaults to ``$__object_id``.
exclude exclude
Pattern of files which are excluded from removal. Pattern of files which are excluded from removal.
@ -46,6 +49,11 @@ EXAMPLES
--exclude '.+\(charset\.conf\|security\.conf\)' \ --exclude '.+\(charset\.conf\|security\.conf\)' \
--onchange 'service apache2 restart' --onchange 'service apache2 restart'
__clean_path apache2-conf-enabled \
--path /etc/apache2/conf-enabled \
--pattern '.+' \
--exclude '.+\(charset\.conf\|security\.conf\)' \
--onchange 'service apache2 restart'
AUTHORS AUTHORS
------- -------

View file

@ -1,2 +1,3 @@
exclude exclude
onchange onchange
path

View file

@ -0,0 +1,9 @@
#!/bin/sh -e
# shellcheck disable=SC1090
file="$( . "$__type_explorer/file" )"
if [ -f "$file" ]
then
cat "$file"
fi

View file

@ -60,6 +60,9 @@ nofile
Don't manage existence, ownership and permissions of the the authorized_keys Don't manage existence, ownership and permissions of the the authorized_keys
file. file.
remove-unknown
Remove undefined keys.
EXAMPLES EXAMPLES
-------- --------
@ -70,6 +73,12 @@ EXAMPLES
__ssh_authorized_keys root \ __ssh_authorized_keys root \
--key "$(cat ~/.ssh/id_rsa.pub)" --key "$(cat ~/.ssh/id_rsa.pub)"
# same as above, but make sure your key is only key in
# root's authorized_keys file
__ssh_authorized_keys root \
--key "$(cat ~/.ssh/id_rsa.pub)" \
--remove-unknown
# allow key to login as user-name # allow key to login as user-name
__ssh_authorized_keys user-name \ __ssh_authorized_keys user-name \
--key "ssh-rsa AXYZAAB3NzaC1yc2..." --key "ssh-rsa AXYZAAB3NzaC1yc2..."

View file

@ -55,8 +55,12 @@ _cksum() {
echo "$1" | cksum | cut -d' ' -f 1 echo "$1" | cksum | cut -d' ' -f 1
} }
_type_and_key() {
echo "$1" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }'
}
while read -r key; do while read -r key; do
type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" type_and_key="$( _type_and_key "$key" )"
object_id="$(_cksum "$file")-$(_cksum "$type_and_key")" object_id="$(_cksum "$file")-$(_cksum "$type_and_key")"
set -- "$object_id" set -- "$object_id"
set -- "$@" --file "$file" set -- "$@" --file "$file"
@ -72,3 +76,24 @@ while read -r key; do
# Ensure __ssh_authorized_key does not read stdin # Ensure __ssh_authorized_key does not read stdin
__ssh_authorized_key "$@" < /dev/null __ssh_authorized_key "$@" < /dev/null
done < "$__object/parameter/key" done < "$__object/parameter/key"
if [ -f "$__object/parameter/remove-unknown" ] &&
[ -s "$__object/explorer/keys" ]
then
while read -r key
do
type_and_key="$( _type_and_key "$key" )"
if grep -Fq "$type_and_key" "$__object/parameter/key"
then
continue
fi
__ssh_authorized_key "remove-$( _cksum "$file$key" )" \
--file "$file" \
--key "$key" \
--state absent \
< /dev/null
done \
< "$__object/explorer/keys"
fi

View file

@ -1,2 +1,3 @@
noparent noparent
nofile nofile
remove-unknown

View file

@ -29,10 +29,14 @@ import time
import itertools import itertools
import tempfile import tempfile
import multiprocessing import multiprocessing
from cdist.mputil import mp_pool_run, mp_sig_handler
import atexit import atexit
import shutil import shutil
import socket import socket
from cdist.mputil import mp_pool_run, mp_sig_handler
from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts
import cdist import cdist
import cdist.hostsource import cdist.hostsource
import cdist.exec.local import cdist.exec.local
@ -40,8 +44,6 @@ import cdist.exec.remote
import cdist.util.ipaddr as ipaddr import cdist.util.ipaddr as ipaddr
import cdist.util.python_type_util as pytype_util import cdist.util.python_type_util as pytype_util
import cdist.configuration import cdist.configuration
from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts
def graph_check_cycle(graph): def graph_check_cycle(graph):
@ -198,7 +200,6 @@ class Config(object):
@classmethod @classmethod
def commandline(cls, args): def commandline(cls, args):
"""Configure remote system""" """Configure remote system"""
if (args.parallel and args.parallel != 1) or args.jobs: if (args.parallel and args.parallel != 1) or args.jobs:
if args.timestamp: if args.timestamp:
cdist.log.setupTimestampingParallelLogging() cdist.log.setupTimestampingParallelLogging()
@ -206,6 +207,7 @@ class Config(object):
cdist.log.setupParallelLogging() cdist.log.setupParallelLogging()
elif args.timestamp: elif args.timestamp:
cdist.log.setupTimestampingLogging() cdist.log.setupTimestampingLogging()
log = logging.getLogger("config") log = logging.getLogger("config")
# No new child process if only one host at a time. # No new child process if only one host at a time.
@ -384,10 +386,16 @@ class Config(object):
If operating in parallel then return tuple (host, True|False, ) If operating in parallel then return tuple (host, True|False, )
so that main process knows for which host function was successful. so that main process knows for which host function was successful.
""" """
log = logging.getLogger(host) log = logging.getLogger(host)
try: try:
if args.log_server:
# Start a log server so that nested `cdist config` runs
# have a place to send their logs to.
log_server_socket_dir = tempfile.mkdtemp()
cls._register_path_for_removal(log_server_socket_dir)
cdist.log.setupLogServer(log_server_socket_dir, log)
remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds(
args) args)
log.debug("remote_exec for host \"{}\": {}".format( log.debug("remote_exec for host \"{}\": {}".format(

View file

@ -27,6 +27,7 @@ import cdist.argparse
import re import re
import multiprocessing import multiprocessing
import logging import logging
import sys
class Singleton(type): class Singleton(type):
@ -246,9 +247,33 @@ class LogLevelOption(OptionBase):
return VerbosityOption().translate(val) return VerbosityOption().translate(val)
class ColoredOutputOption(BooleanOption):
CHOICES = ('always', 'never', 'auto')
DEFAULT = 'auto'
def get_converter(self):
return self.translate
@staticmethod
def translate(val):
if isinstance(val, bool):
return val
elif val == 'always':
return True
elif val == 'never':
return False
elif val == 'auto':
return 'NO_COLOR' not in os.environ and sys.stdout.isatty()
ColoredOutputOption.DEFAULT = ColoredOutputOption.translate(
ColoredOutputOption.DEFAULT)
_ARG_OPTION_MAPPING = { _ARG_OPTION_MAPPING = {
'beta': 'beta', 'beta': 'beta',
'cache_path_pattern': 'cache_path_pattern', 'cache_path_pattern': 'cache_path_pattern',
'colored_output': 'colored_output',
'conf_dir': 'conf_dir', 'conf_dir': 'conf_dir',
'manifest': 'init_manifest', 'manifest': 'init_manifest',
'out_path': 'out_path', 'out_path': 'out_path',
@ -294,6 +319,7 @@ class Configuration(metaclass=Singleton):
'remote_shell': StringOption('remote_shell'), 'remote_shell': StringOption('remote_shell'),
'cache_path_pattern': StringOption('cache_path_pattern'), 'cache_path_pattern': StringOption('cache_path_pattern'),
'conf_dir': ConfDirOption(), 'conf_dir': ConfDirOption(),
'colored_output': ColoredOutputOption('colored_output'),
'init_manifest': StringOption('init_manifest'), 'init_manifest': StringOption('init_manifest'),
'out_path': StringOption('out_path'), 'out_path': StringOption('out_path'),
'remote_out_path': StringOption('remote_out_path'), 'remote_out_path': StringOption('remote_out_path'),
@ -319,6 +345,7 @@ class Configuration(metaclass=Singleton):
'CDIST_REMOTE_COPY': 'remote_copy', 'CDIST_REMOTE_COPY': 'remote_copy',
'CDIST_INVENTORY_DIR': 'inventory_dir', 'CDIST_INVENTORY_DIR': 'inventory_dir',
'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern', 'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern',
'CDIST_COLORED_OUTPUT': 'colored_output',
'__cdist_log_level': 'verbosity', '__cdist_log_level': 'verbosity',
} }
ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', ) ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', )
@ -327,11 +354,10 @@ class Configuration(metaclass=Singleton):
} }
ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING
ADJUST_ARG_OPTION_MAPPING = { ADJUST_ARG_OPTION_MAPPING = {v: k for k, v in _ARG_OPTION_MAPPING.items()}
_ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING
}
REQUIRED_DEFAULT_CONFIG_VALUES = { REQUIRED_DEFAULT_CONFIG_VALUES = {
'GLOBAL': { 'GLOBAL': {
'colored_output': 'auto',
'verbosity': 0, 'verbosity': 0,
}, },
} }
@ -484,8 +510,7 @@ class Configuration(metaclass=Singleton):
newconfig = self._read_config_file(config_file) newconfig = self._read_config_file(config_file)
self._update_config_dict(config, newconfig) self._update_config_dict(config, newconfig)
# command line config file # command line config file
if (self.args and 'config_file' in self.args and if (self.args and self.args.get('config_file', None)):
self.args['config_file']):
newconfig = self._read_config_file(self.args['config_file']) newconfig = self._read_config_file(self.args['config_file'])
self._update_config_dict(config, newconfig) self._update_config_dict(config, newconfig)
# command line # command line

View file

@ -118,6 +118,10 @@ class Code(object):
if dry_run: if dry_run:
self.env['__cdist_dry_run'] = '1' self.env['__cdist_dry_run'] = '1'
if '__cdist_log_server_socket_export' in os.environ:
self.env['__cdist_log_server_socket'] = os.environ[
'__cdist_log_server_socket_export']
def run_py(self, cdist_object): def run_py(self, cdist_object):
cdist_type = cdist_object.cdist_type cdist_type = cdist_object.cdist_type
type_class = get_pytype_class(cdist_type) type_class = get_pytype_class(cdist_type)

View file

@ -119,6 +119,8 @@ class Manifest(object):
'__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level': util.log_level_env_var_val(self.log),
'__cdist_log_level_name': util.log_level_name_env_var_val( '__cdist_log_level_name': util.log_level_name_env_var_val(
self.log), self.log),
'__cdist_colored_log': str(
cdist.log.CdistFormatter.USE_COLORS).lower(),
} }
if dry_run: if dry_run:

View file

@ -152,6 +152,9 @@ class Emulator(object):
# if invalid __cdist_log_level value # if invalid __cdist_log_level value
logging.root.setLevel(logging.WARNING) logging.root.setLevel(logging.WARNING)
colored_log = self.env.get('__cdist_colored_log', 'false')
cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
self.log = logging.getLogger(self.target_host[0]) self.log = logging.getLogger(self.target_host[0])
def get_args_parser(self): def get_args_parser(self):

View file

@ -22,6 +22,7 @@
import fcntl import fcntl
import logging import logging
import os import os
import cdist.log
log = logging.getLogger('cdist-flock') log = logging.getLogger('cdist-flock')

View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2013 Steven Armstrong (steven-cdist at armstrong.cc) # 2013-2019 Steven Armstrong (steven-cdist at armstrong.cc)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -22,9 +22,21 @@
import cdist.config import cdist.config
import cdist.core import cdist.core
import cdist.log
class Install(cdist.config.Config): class Install(cdist.config.Config):
@classmethod
def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
parallel, configuration, remove_remote_files_dirs=False):
# Always start log server during cdist install so that nested
# `cdist config` runs have a place to send their logs to.
args.log_server = True
super().onehost(host, host_tags, host_base_path, host_dir_name, args,
parallel, configuration, remove_remote_files_dirs)
def object_list(self): def object_list(self):
"""Short name for object list retrieval. """Short name for object list retrieval.
In install mode, we only care about install objects. In install mode, we only care about install objects.

View file

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
# 2019-2020 Steven Armstrong
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -20,9 +21,16 @@
# #
# #
import logging
import sys
import datetime import datetime
import logging
import logging.handlers
import sys
import os
import asyncio
import contextlib
import pickle
import struct
import threading
# Define additional cdist logging levels. # Define additional cdist logging levels.
@ -50,9 +58,32 @@ def _trace(msg, *args, **kwargs):
logging.trace = _trace logging.trace = _trace
class DefaultLog(logging.Logger): class CdistFormatter(logging.Formatter):
USE_COLORS = False
RESET = '\033[0m'
COLOR_MAP = {
'ERROR': '\033[0;31m',
'WARNING': '\033[0;33m',
'INFO': '\033[0;94m',
'VERBOSE': '\033[0;34m',
'DEBUG': '\033[0;90m',
'TRACE': '\033[0;37m',
}
FORMAT = '%(levelname)s: %(message)s' def __init__(self, fmt):
super().__init__(fmt=fmt)
def format(self, record):
msg = super().format(record)
if self.USE_COLORS:
color = self.COLOR_MAP.get(record.levelname)
if color:
msg = color + msg + self.RESET
return msg
class DefaultLog(logging.Logger):
FORMAT = '%(levelname)s: %(name)s: %(message)s'
class StdoutFilter(logging.Filter): class StdoutFilter(logging.Filter):
def filter(self, rec): def filter(self, rec):
@ -66,9 +97,13 @@ class DefaultLog(logging.Logger):
super().__init__(name) super().__init__(name)
self.propagate = False self.propagate = False
formatter = logging.Formatter(self.FORMAT) if '__cdist_log_server_socket' in os.environ:
log_server_socket = os.environ['__cdist_log_server_socket']
self.addFilter(self) socket_handler = logging.handlers.SocketHandler(log_server_socket,
None)
self.addHandler(socket_handler)
else:
formatter = CdistFormatter(self.FORMAT)
stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler = logging.StreamHandler(sys.stdout)
stdout_handler.addFilter(self.StdoutFilter()) stdout_handler.addFilter(self.StdoutFilter())
@ -83,13 +118,6 @@ class DefaultLog(logging.Logger):
self.addHandler(stdout_handler) self.addHandler(stdout_handler)
self.addHandler(stderr_handler) self.addHandler(stderr_handler)
def filter(self, record):
"""Prefix messages with logger name"""
record.msg = self.name + ": " + str(record.msg)
return True
def verbose(self, msg, *args, **kwargs): def verbose(self, msg, *args, **kwargs):
self.log(logging.VERBOSE, msg, *args, **kwargs) self.log(logging.VERBOSE, msg, *args, **kwargs)
@ -111,7 +139,7 @@ class TimestampingLog(DefaultLog):
class ParallelLog(DefaultLog): class ParallelLog(DefaultLog):
FORMAT = '%(levelname)s: [%(process)d]: %(message)s' FORMAT = '%(levelname)s: [%(process)d]: %(name)s: %(message)s'
class TimestampingParallelLog(TimestampingLog, ParallelLog): class TimestampingParallelLog(TimestampingLog, ParallelLog):
@ -138,4 +166,42 @@ def setupParallelLogging():
logging.setLoggerClass(ParallelLog) logging.setLoggerClass(ParallelLog)
async def handle_log_client(reader, writer):
while True:
chunk = await reader.read(4)
if len(chunk) < 4:
return
data_size = struct.unpack('>L', chunk)[0]
data = await reader.read(data_size)
obj = pickle.loads(data)
record = logging.makeLogRecord(obj)
logger = logging.getLogger(record.name)
logger.handle(record)
def run_log_server(server_address):
# Get a new loop inside the current thread to run the log server.
loop = asyncio.new_event_loop()
loop.create_task(asyncio.start_unix_server(handle_log_client,
server_address))
loop.run_forever()
def setupLogServer(socket_dir, log=logging.getLogger(__name__)):
"""Run a asyncio based unix socket log server in a background thread.
"""
log_server_socket = os.path.join(socket_dir, 'log-server')
log.debug('Starting logging server on: %s', log_server_socket)
os.environ['__cdist_log_server_socket_export'] = log_server_socket
with contextlib.suppress(FileNotFoundError):
os.remove(log_server_socket)
t = threading.Thread(target=run_log_server, args=(log_server_socket,))
# Deamonizing the thread means we don't have to care about stoping it.
# It will die together with the main process.
t.daemon = True
t.start()
setupDefaultLogging() setupDefaultLogging()

View file

@ -7,6 +7,7 @@ from docutils.io import FileOutput
from os import path from os import path
from sphinx.util.nodes import inline_all_toctrees from sphinx.util.nodes import inline_all_toctrees
from sphinx import addnodes from sphinx import addnodes
from sphinx.util import logging
""" """
Extension based on sphinx builtin manpage. Extension based on sphinx builtin manpage.
@ -15,6 +16,9 @@ from sphinx import addnodes
""" """
logger = logging.getLogger(__name__)
class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator):
def header(self): def header(self):
@ -28,7 +32,7 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter):
def __init__(self, builder): def __init__(self, builder):
super().__init__(builder) super().__init__(builder)
self.translator_class = ( self.translator_class = (
self.builder.translator_class or ManualPageTranslator) self.builder.get_translator_class() or ManualPageTranslator)
class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
@ -43,7 +47,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
components=(docwriter,), components=(docwriter,),
read_config_files=True).get_default_values() read_config_files=True).get_default_values()
self.info(bold('writing... '), nonl=True) logger.info(bold('writing... '), nonl=True)
for info in self.config.man_pages: for info in self.config.man_pages:
docname, name, description, authors, section = info docname, name, description, authors, section = info
@ -54,7 +58,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
authors = [] authors = []
targetname = '%s.%s' % (name, section) targetname = '%s.%s' % (name, section)
self.info(darkgreen(targetname) + ' { ', nonl=True) logger.info(darkgreen(targetname) + ' { ', nonl=True)
destination = FileOutput( destination = FileOutput(
destination_path=path.join(self.outdir, targetname), destination_path=path.join(self.outdir, targetname),
encoding='utf-8') encoding='utf-8')
@ -63,7 +67,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
docnames = set() docnames = set()
largetree = inline_all_toctrees(self, docnames, docname, tree, largetree = inline_all_toctrees(self, docnames, docname, tree,
darkgreen, [docname]) darkgreen, [docname])
self.info('} ', nonl=True) logger.info('} ', nonl=True)
self.env.resolve_references(largetree, docname, self) self.env.resolve_references(largetree, docname, self)
# remove pending_xref nodes # remove pending_xref nodes
for pendingnode in largetree.traverse(addnodes.pending_xref): for pendingnode in largetree.traverse(addnodes.pending_xref):
@ -76,7 +80,7 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder):
largetree.settings.section = section largetree.settings.section = section
docwriter.write(largetree, destination) docwriter.write(largetree, destination)
self.info() logger.info("")
def setup(app): def setup(app):

View file

@ -28,10 +28,12 @@ import argparse
from cdist import test from cdist import test
import cdist.argparse as cap import cdist.argparse as cap
import logging import logging
import sys
my_dir = op.abspath(op.dirname(__file__)) my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures') fixtures = op.join(my_dir, 'fixtures')
interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") interpolation_config_file = op.join(fixtures, "interpolation-test.cfg")
colored_output_default = 'auto'
def newConfigParser(): def newConfigParser():
@ -153,6 +155,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/bin/sh', 'remote_shell': '/bin/sh',
'inventory_dir': '', 'inventory_dir': '',
'cache_path_pattern': '', 'cache_path_pattern': '',
'colored_output': colored_output_default,
'conf_dir': '', 'conf_dir': '',
'init_manifest': '', 'init_manifest': '',
'out_path': '', 'out_path': '',
@ -184,6 +187,8 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/bin/sh', 'remote_shell': '/bin/sh',
'inventory_dir': None, 'inventory_dir': None,
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': cc.ColoredOutputOption.translate(
colored_output_default),
'conf_dir': None, 'conf_dir': None,
'init_manifest': None, 'init_manifest': None,
'out_path': None, 'out_path': None,
@ -390,6 +395,7 @@ class ConfigurationTestCase(test.CdistTestCase):
args = argparse.Namespace() args = argparse.Namespace()
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'verbosity': 0, 'verbosity': 0,
}, },
} }
@ -440,6 +446,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/bin/sh', 'remote_shell': '/bin/sh',
'inventory_dir': None, 'inventory_dir': None,
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': None, 'conf_dir': None,
'init_manifest': None, 'init_manifest': None,
'out_path': None, 'out_path': None,
@ -515,6 +522,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': ['/opt/cdist', ], 'conf_dir': ['/opt/cdist', ],
'init_manifest': None, 'init_manifest': None,
'out_path': None, 'out_path': None,
@ -556,6 +564,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/bin/sh', 'remote_shell': '/bin/sh',
'inventory_dir': '', 'inventory_dir': '',
'cache_path_pattern': '', 'cache_path_pattern': '',
'colored_output': colored_output_default,
'conf_dir': '', 'conf_dir': '',
'init_manifest': '', 'init_manifest': '',
'out_path': '', 'out_path': '',
@ -579,6 +588,8 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': None, 'inventory_dir': None,
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': cc.ColoredOutputOption.translate(
colored_output_default),
'conf_dir': [ 'conf_dir': [
'/opt/cdist/conf', '/opt/cdist/conf',
'/usr/local/share/cdist/conf', '/usr/local/share/cdist/conf',
@ -623,6 +634,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/bin/sh', 'remote_shell': '/bin/sh',
'inventory_dir': '', 'inventory_dir': '',
'cache_path_pattern': '', 'cache_path_pattern': '',
'colored_output': colored_output_default,
'conf_dir': '', 'conf_dir': '',
'init_manifest': '', 'init_manifest': '',
'out_path': '', 'out_path': '',
@ -645,6 +657,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'local_shell': '/usr/bin/sh', 'local_shell': '/usr/bin/sh',
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'colored_output': colored_output_default,
'conf_dir': '/opt/cdist', 'conf_dir': '/opt/cdist',
'remote_copy': 'myscp', 'remote_copy': 'myscp',
'remote_exec': 'myexec', 'remote_exec': 'myexec',
@ -663,6 +676,8 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': cc.ColoredOutputOption.translate(
colored_output_default),
'conf_dir': [ 'conf_dir': [
'/opt/cdist/conf', '/opt/cdist/conf',
'/usr/local/share/cdist/conf', '/usr/local/share/cdist/conf',
@ -694,6 +709,7 @@ class ConfigurationTestCase(test.CdistTestCase):
} }
expected_config = { expected_config = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'verbosity': 0, 'verbosity': 0,
}, },
} }
@ -767,6 +783,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/opt/sysadmin/cdist/inventory', 'inventory_dir': '/opt/sysadmin/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': [ 'conf_dir': [
'/opt/cdist/conf', '/opt/cdist/conf',
'/usr/local/share/cdist/conf', '/usr/local/share/cdist/conf',
@ -865,6 +882,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': [ 'conf_dir': [
'/opt/conf/cdist', '/opt/conf/cdist',
], ],
@ -964,6 +982,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': [ 'conf_dir': [
'/opt/conf/cdist', '/opt/conf/cdist',
], ],
@ -1063,6 +1082,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'remote_shell': '/usr/bin/sh', 'remote_shell': '/usr/bin/sh',
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': [ 'conf_dir': [
'/opt/conf/cdist', '/opt/conf/cdist',
], ],
@ -1095,6 +1115,7 @@ class ConfigurationTestCase(test.CdistTestCase):
'beta': True, 'beta': True,
'inventory_dir': '/var/db/cdist/inventory', 'inventory_dir': '/var/db/cdist/inventory',
'cache_path_pattern': None, 'cache_path_pattern': None,
'colored_output': colored_output_default,
'conf_dir': [ 'conf_dir': [
'/opt/conf/cdist', '/opt/conf/cdist',
], ],
@ -1125,6 +1146,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'inventory_dir': None, 'inventory_dir': None,
'colored_output': colored_output_default,
'conf_dir': None, 'conf_dir': None,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1148,6 +1170,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'verbosity': cap.VERBOSE_DEBUG, 'verbosity': cap.VERBOSE_DEBUG,
}, },
} }
@ -1185,6 +1208,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'save_output_streams': True, 'save_output_streams': True,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1213,6 +1237,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'save_output_streams': False, 'save_output_streams': False,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1241,6 +1266,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'save_output_streams': False, 'save_output_streams': False,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1269,6 +1295,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'save_output_streams': False, 'save_output_streams': False,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1308,6 +1335,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'timestamp': True, 'timestamp': True,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1336,6 +1364,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'timestamp': True, 'timestamp': True,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1364,6 +1393,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'timestamp': False, 'timestamp': False,
'verbosity': 0, 'verbosity': 0,
}, },
@ -1392,6 +1422,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected_config_dict = { expected_config_dict = {
'GLOBAL': { 'GLOBAL': {
'colored_output': colored_output_default,
'timestamp': False, 'timestamp': False,
'verbosity': 0, 'verbosity': 0,
}, },

View file

@ -13,6 +13,14 @@
# Specify cache path pattern. # Specify cache path pattern.
# cache_path_pattern = %h # cache_path_pattern = %h
# #
# colored_output
# Colorize cdist's output. If enabled, cdist will use different colors for
# different log levels.
# Recognized values are 'always', 'never', and 'auto'.
# If the value is 'auto', colors are enabled if stdout is a TTY unless
# the NO_COLOR (https://no-color.org/) environment variable is defined.
# colored_output = auto
#
# conf_dir # conf_dir
# List of configuration directories separated with the character conventionally # List of configuration directories separated with the character conventionally
# used by the operating system to separate search path components (as in PATH), # used by the operating system to separate search path components (as in PATH),

View file

@ -5,6 +5,17 @@ next:
* Core: Add trigger functionality (Nico Schottelius, Darko Poljak) * Core: Add trigger functionality (Nico Schottelius, Darko Poljak)
* Core: Implement core support for python types (Darko Poljak) * Core: Implement core support for python types (Darko Poljak)
6.6.0: 2020-06-17
* Type __ssh_authorized_keys: Add option for removing undefined keys (Ander Punnar)
* Core: Support colored log output (Evil Ham)
* Core: Tune colored log output and respect NO_COLOR env var (Dennis Camera)
* Documentation: Fix failing man pages build with newer sphinx versions (Darko Poljak)
* Documentation: Fix trivial grammatical mistakes (Jaak Ristioja)
* Explorer os: Fix for sles15 (Daniel Heule)
* Type __clean_path: Add --path parameter (Ander Punnar)
* Core: Increase minimal supported Python version to 3.5 (Darko Poljak)
* Core: Add log server for nested logging (Steven Armstrong)
6.5.6: 2020-05-25 6.5.6: 2020-05-25
* Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius) * Type __pyvenv: Switch to python3 -m venv for Ubuntu (Nico Schottelius)
* Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius) * Type __letsencrypt_cert: Whitelist Ubuntu (Nico Schottelius)

View file

@ -25,7 +25,7 @@ people, have a look at `cdist best practice <cdist-best-practice.html>`_.
Setup working directory and branch Setup working directory and branch
---------------------------------- ----------------------------------
I assume you have a fresh copy of the cdist tree in ~/cdist, cloned from I assume you have a fresh copy of the cdist tree in ~/cdist, cloned from
one of the official urls (see `cdist quickstart <cdist-quickstart.html>`_ if you don't). one of the official URLs (see `cdist quickstart <cdist-quickstart.html>`_ if you don't).
Entering the command "git branch" should show you "* master", which indicates Entering the command "git branch" should show you "* master", which indicates
you are on the **master** branch. you are on the **master** branch.

View file

@ -59,7 +59,7 @@ typeorder
Object cache overview Object cache overview
~~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~~
Each object under :strong:`object` directory has its own structurue. Each object under :strong:`object` directory has its own structure.
code-local code-local
code generated from gencode-local, present only if something is code generated from gencode-local, present only if something is

View file

@ -21,7 +21,7 @@ precedence. Configuration option value read from source with higher
precedence will overwrite option value, if exists, read from source with precedence will overwrite option value, if exists, read from source with
lower precedence. That means that command-line option wins them all. lower precedence. That means that command-line option wins them all.
Users can decide on the local conifguration file location. It can be either Users can decide on the local configuration file location. It can be either
~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist, ~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist,
then ~/.cdist.cfg is used. then ~/.cdist.cfg is used.

View file

@ -9,15 +9,15 @@ Source Host
This is the machine from which you will configure target hosts. This is the machine from which you will configure target hosts.
* /bin/sh: A posix like shell (for instance bash, dash, zsh) * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
* Python >= 3.2 * Python >= 3.5
* SSH client * SSH client
* sphinx (for building html docs and/or the man pages) * sphinx (for building html docs and/or the man pages)
Target Hosts Target Hosts
~~~~~~~~~~~~ ~~~~~~~~~~~~
* /bin/sh: A posix like shell (for instance bash, dash, zsh) * /bin/sh: A POSIX like shell (for instance bash, dash, zsh)
* SSH server * SSH server
Install cdist Install cdist

View file

@ -198,7 +198,7 @@ We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$u
Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**. Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**.
If this directory does not exist then **__file** type would error. If this directory does not exist then **__file** type would error.
We also use stdin as file content source. For details see `Input from stdin <cdist-type.html#input-from-stdin>`_. We also use stdin as file content source. For details see `Input from stdin <cdist-type.html#input-from-stdin>`_.
For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent For feeding stdin we use here-document (**<<** operator). It allows redirection of subsequent
lines read by the shell to the input of a command until a line containing only the delimiter lines read by the shell to the input of a command until a line containing only the delimiter
and a newline, with no blank characters in between (EOF in our case). and a newline, with no blank characters in between (EOF in our case).
@ -546,7 +546,7 @@ we have changed our **wsgi.py** file uWSGI reloads the application.
Our application selects and lists items from **items** table. Our application selects and lists items from **items** table.
Openning application Opening application
~~~~~~~~~~~~~~~~~~~~ ~~~~~~~~~~~~~~~~~~~~
Finally try the application:: Finally try the application::

View file

@ -222,65 +222,89 @@ __cdist_log_level, __cdist_log_level_name
| TRACE | 5 | | TRACE | 5 |
+----------------+-----------------+ +----------------+-----------------+
Available for: initial manifest, explorer, type manifest, type explorer,
type gencode.
__cdist_colored_log
whether or not cdist's log has colors enabled.
Is set to the string ``true`` if cdist's output is using colors,
otherwise the variable contains the string ``false``.
Available for: initial manifest, explorer, type manifest, type explorer, Available for: initial manifest, explorer, type manifest, type explorer,
type gencode. type gencode.
__cdist_dry_run __cdist_dry_run
Is set only when doing dry run (``-n`` flag). Is set only when doing dry run (``-n`` flag).
Available for: initial manifest, explorer, type manifest, type explorer, Available for: initial manifest, explorer, type manifest, type explorer,
type gencode. type gencode.
__explorer __explorer
Directory that contains all global explorers. Directory that contains all global explorers.
Available for: initial manifest, explorer, type explorer, shell. Available for: initial manifest, explorer, type explorer, shell.
__files __files
Directory that contains content from the "files" subdirectories Directory that contains content from the "files" subdirectories
from the configuration directories. from the configuration directories.
Available for: initial manifest, type manifest, type gencode, shell. Available for: initial manifest, type manifest, type gencode, shell.
__manifest __manifest
Directory that contains the initial manifest. Directory that contains the initial manifest.
Available for: initial manifest, type manifest, shell. Available for: initial manifest, type manifest, shell.
__global __global
Directory that contains generic output like explorer. Directory that contains generic output like explorer.
Available for: initial manifest, type manifest, type gencode, shell. Available for: initial manifest, type manifest, type gencode, shell.
__messages_in __messages_in
File to read messages from. File to read messages from.
Available for: initial manifest, type manifest, type gencode. Available for: initial manifest, type manifest, type gencode.
__messages_out __messages_out
File to write messages. File to write messages.
Available for: initial manifest, type manifest, type gencode. Available for: initial manifest, type manifest, type gencode.
__object __object
Directory that contains the current object. Directory that contains the current object.
Available for: type manifest, type explorer, type gencode and code scripts. Available for: type manifest, type explorer, type gencode and code scripts.
__object_id __object_id
The type unique object id. The type unique object id.
Available for: type manifest, type explorer, type gencode and code scripts. Available for: type manifest, type explorer, type gencode and code scripts.
Note: The leading and the trailing "/" will always be stripped (caused by
| Note: The leading and the trailing "/" will always be stripped (caused by
the filesystem database and ensured by the core). the filesystem database and ensured by the core).
Note: Double slashes ("//") will not be fixed and result in an error. | Note: Double slashes ("//") will not be fixed and result in an error.
__object_name __object_name
The full qualified name of the current object. The full qualified name of the current object.
Available for: type manifest, type explorer, type gencode. Available for: type manifest, type explorer, type gencode.
__target_host __target_host
The host we are deploying to. This is primary variable. It's content is The host we are deploying to. This is primary variable. It's content is
literally the one user passed in. literally the one user passed in.
Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
__target_hostname __target_hostname
The hostname of host we are deploying to. This variable is derived from The hostname of host we are deploying to. This variable is derived from
**__target_host** (using **socket.getaddrinfo(__target_host)** and then **__target_host** (using **socket.getaddrinfo(__target_host)** and then
**socket.gethostbyaddr()**). **socket.gethostbyaddr()**).
Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
__target_fqdn __target_fqdn
The fully qualified domain name of the host we are deploying to. The fully qualified domain name of the host we are deploying to.
This variable is derived from **__target_host** This variable is derived from **__target_host**
(using **socket.getfqdn()**). (using **socket.getfqdn()**).
Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
__target_host_tags __target_host_tags
Comma separated list of target host tags. Comma separated list of target host tags.
Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
__type __type
Path to the current type. Path to the current type.
Available for: type manifest, type gencode. Available for: type manifest, type gencode.
__type_explorer __type_explorer
Directory that contains the type explorers. Directory that contains the type explorers.
Available for: type explorer. Available for: type explorer.
Environment variables (for writing) Environment variables (for writing)
@ -344,6 +368,11 @@ CDIST_INVENTORY_DIR
CDIST_BETA CDIST_BETA
Enable beta functionalities. Enable beta functionalities.
CDIST_COLORED_OUTPUT
Colorize cdist's output. If enabled, cdist will use different colors for
different log levels.
Recognized values are 'always', 'never', and 'auto' (the default).
CDIST_CACHE_PATH_PATTERN CDIST_CACHE_PATH_PATTERN
Custom cache path pattern. Custom cache path pattern.
eof eof

View file

@ -14,7 +14,7 @@ To upgrade cdist in the current branch use
make man make man
export MANPATH=$MANPATH:$(pwd -P)/doc/man export MANPATH=$MANPATH:$(pwd -P)/doc/man
If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break. If you stay on a version branch (i.e. 1.0, 1.1., ...), nothing should break.
The master branch on the other hand is the development branch and may not be The master branch on the other hand is the development branch and may not be
working, break your setup or eat the tree in your garden. working, break your setup or eat the tree in your garden.
@ -61,7 +61,7 @@ After that, you can go back and continue the upgrade:
Update the python package Update the python package
------------------------- -------------------------
To upgrade to the lastet version do To upgrade to the latest version do
.. code-block:: sh .. code-block:: sh
@ -158,7 +158,7 @@ Updating from 1.5 to 1.6
* If you used **\_\_package_apt --preseed**, you need to use the new * If you used **\_\_package_apt --preseed**, you need to use the new
type **\_\_debconf_set_selections** instead. type **\_\_debconf_set_selections** instead.
* The **\_\_package** types accepted either --state deinstalled or * The **\_\_package** types accepted either --state deinstalled or
--state uninstaaled. Starting with 1.6, it was made consistently --state uninstalled. Starting with 1.6, it was made consistently
to --state removed. to --state removed.
Updating from 1.3 to 1.5 Updating from 1.3 to 1.5

View file

@ -21,7 +21,7 @@ Not only is shell scripting widely known by system engineers,
but it is also a very powerful language. Here are some features but it is also a very powerful language. Here are some features
which make daily work easy: which make daily work easy:
* Configuration can react dynamicly on explored values * Configuration can react dynamically on explored values
* High level string manipulation (using sed, awk, grep) * High level string manipulation (using sed, awk, grep)
* Conditional support (**if, case**) * Conditional support (**if, case**)
* Loop support (**for, while**) * Loop support (**for, while**)
@ -44,7 +44,7 @@ Cdist requires very little on a target system. Even better,
in almost all cases all dependencies are usually fulfilled. in almost all cases all dependencies are usually fulfilled.
Cdist does not require an agent or high level programming Cdist does not require an agent or high level programming
languages on the target host: it will run on any host that languages on the target host: it will run on any host that
has a **ssh server running** and a posix compatible shell has a **ssh server running** and a POSIX compatible shell
(**/bin/sh**). Compared to other configuration management systems, (**/bin/sh**). Compared to other configuration management systems,
it does not require to open up an additional port. it does not require to open up an additional port.

View file

@ -15,52 +15,51 @@ SYNOPSIS
cdist banner [-h] [-l LOGLEVEL] [-q] [-v] cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
[-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
[-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
[-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
[-p [HOST_MAX]] [-s] [-t] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]] [host [host ...]]
cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-g CONFIG_FILE] [-4] [-6] [-C CACHE_PATH_PATTERN]
[-j [JOBS]] [-n] [-o OUT_PATH] [-P] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [--log-server]
[-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [-n] [-o OUT_PATH] [-P] [-R [{tar,tgz,tbz2,txz}]]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY]
[-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [--remote-exec REMOTE_EXEC] [-S] [-I INVENTORY_DIR] [-A]
[-p [HOST_MAX]] [-s] [-t] [-a] [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]] [host [host ...]]
cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ...
cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-g CONFIG_FILE] [-I INVENTORY_DIR] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE]
[-f HOSTFILE]
[host [host ...]] [host [host ...]]
cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-g CONFIG_FILE] [-I INVENTORY_DIR] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-f HOSTFILE]
[-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] [-T TAGFILE] [-t TAGLIST]
[host [host ...]] [host [host ...]]
cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE] [-f HOSTFILE]
[host [host ...]] [host [host ...]]
cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
[host [host ...]] [host [host ...]]
cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [--colors WHEN]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
[-H] [-t]
[host [host ...]] [host [host ...]]
cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] cdist preos [-h] [-l LOGLEVEL] [-q] [-v] [-c CONF_DIR] [-g CONFIG_FILE] [-L]
[-L]
[preos] ... [preos] ...
cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B] cdist preos [preos-options] debian [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-a ARCH] [-B]
@ -87,9 +86,10 @@ SYNOPSIS
[-y REMOTE_COPY] [-y REMOTE_COPY]
target_dir target_dir
cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [--colors WHEN] [-s SHELL]
cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t] [pattern] cdist info [-h] [-a] [-c CONF_DIR] [-e] [-F] [-f] [-g CONFIG_FILE] [-t]
[pattern]
cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] cdist trigger [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4]
[-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
@ -105,7 +105,7 @@ cdist is the frontend executable to the cdist configuration management.
It supports different subcommands as explained below. It supports different subcommands as explained below.
It is written in Python so it requires :strong:`python`\ (1) to be installed. It is written in Python so it requires :strong:`python`\ (1) to be installed.
It requires a minimal Python version 3.2. It requires a minimal Python version 3.5.
GENERAL GENERAL
------- -------
@ -114,6 +114,14 @@ All commands accept the following options:
**-h, --help** **-h, --help**
Show the help screen. Show the help screen.
**--colors WHEN**
Colorize cdist's output. If enabled, cdist will use different colors for
different log levels.
WHEN recognizes the values 'always', 'never', and 'auto' (the default).
If the value is 'auto', colored output is enabled if stdout is a TTY
unless the NO_COLOR (https://no-color.org/) environment variable is defined.
**-l LOGLEVEL, --log-level LOGLEVEL** **-l LOGLEVEL, --log-level LOGLEVEL**
Set the specified verbosity level. The levels, in Set the specified verbosity level. The levels, in
order from the lowest to the highest, are: ERROR (-1), order from the lowest to the highest, are: ERROR (-1),
@ -204,6 +212,12 @@ Install command is currently in beta.
are supported. Without argument CPU count is used by are supported. Without argument CPU count is used by
default. default.
**--log-server**
Start a log server for sub processes to use. This is
mainly useful when running cdist nested from a code-
local script. Log server is always implicitly started
for 'install' command.
**-n, --dry-run** **-n, --dry-run**
Do not execute code. Do not execute code.
@ -780,6 +794,9 @@ The possible keywords and their meanings are as follows:
:strong:`cache_path_pattern` :strong:`cache_path_pattern`
Specify cache path pattern. Specify cache path pattern.
:strong:`colored_output`
Colorize cdist's output. cf. the :code:`--colors` option.
:strong:`conf_dir` :strong:`conf_dir`
List of configuration directories separated with the character conventionally List of configuration directories separated with the character conventionally
used by the operating system to separate search path components (as in PATH), used by the operating system to separate search path components (as in PATH),
@ -1008,6 +1025,9 @@ CDIST_BETA
CDIST_CACHE_PATH_PATTERN CDIST_CACHE_PATH_PATTERN
Custom cache path pattern. Custom cache path pattern.
CDIST_COLORED_OUTPUT
Colorize cdist's output. cf. the :code:`--colors` option.
CDIST_CONFIG_FILE CDIST_CONFIG_FILE
Custom configuration file. Custom configuration file.
@ -1070,5 +1090,5 @@ such case and display a warning message. An example of such a case:
COPYING COPYING
------- -------
Copyright \(C) 2011-2019 Nico Schottelius. Free use of this software is Copyright \(C) 2011-2020 Nico Schottelius. Free use of this software is
granted under the terms of the GNU General Public License v3 or later (GPLv3+). granted under the terms of the GNU General Public License v3 or later (GPLv3+).

View file

@ -60,10 +60,9 @@ def commandline():
if __name__ == "__main__": if __name__ == "__main__":
cdistpythonversion = '3.5' if sys.version < cdist.MIN_SUPPORTED_PYTHON_VERSION:
if sys.version < cdistpythonversion:
print('Python >= {} is required on the source host.'.format( print('Python >= {} is required on the source host.'.format(
cdistpythonversion), file=sys.stderr) cdist.MIN_SUPPORTED_PYTHON_VERSIO), file=sys.stderr)
sys.exit(1) sys.exit(1)
exit_code = 0 exit_code = 0