mark/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests Projects Releases Wiki Activity

Add -l/--log-level option. Honor __cdist_loglevel env var. (#572)

Browse source 
Add -l/--log-level option, __cdist_loglevel -> __cdist_log_level; honor __cdist_log_level env var
This commit is contained in:
Darko Poljak 2017-09-09 21:17:29 +02:00 committed by GitHub
parent f08ac264a0
commit 3454da076f
23 changed files with 152 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
cdist/argparse.py
View file

@ -3,6 +3,7 @@ import cdist
import multiprocessing
import logging
import collections
import functools
import cdist.configuration
@ -72,15 +73,15 @@ def check_beta(args_dict):
raise cdist.CdistBetaRequired(cmd, arg)
def check_positive_int(value):
def check_lower_bounded_int(value, lower_bound, name):
try:
val = int(value)
except ValueError:
raise argparse.ArgumentTypeError(
"{} is invalid int value".format(value))
if val <= 0:
if val < lower_bound:
raise argparse.ArgumentTypeError(
"{} is invalid positive int value".format(val))
"{} is invalid {} value".format(val, name))
return val
@ -94,6 +95,17 @@ def get_parsers():
parser = {}
# Options _all_ parsers have in common
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['loglevel'].add_argument(
'-l', '--log-level', metavar='LOGLEVEL',
type=functools.partial(check_lower_bounded_int, lower_bound=-1,
name="log level"),
help=('Set the specified verbosity level. '
'The levels, in order from the lowest to the highest, are: '
'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
'TRACE (4 or higher). If used along with -v then -v '
'increases last set value and -l overwrites last set '
'value.'),
action='store', dest='verbose', required=False)
parser['loglevel'].add_argument(
'-q', '--quiet',
help='Quiet mode: disables logging, including WARNING and ERROR.',
@ -105,7 +117,9 @@ def get_parsers():
'is 0 which includes ERROR and WARNING levels. '
'The levels, in order from the lowest to the highest, are: '
'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) '
'TRACE (4 or higher).'),
'TRACE (4 or higher). If used along with -l then -l '
'overwrites last set value and -v increases last set '
'value.'),
action='count', default=None)
parser['beta'] = argparse.ArgumentParser(add_help=False)
@ -164,7 +178,8 @@ def get_parsers():
dest='manifest', required=False)
parser['config_main'].add_argument(
'-j', '--jobs', nargs='?',
type=check_positive_int,
type=functools.partial(check_lower_bounded_int, lower_bound=1,
name="positive int"),
help=('Operate in parallel in specified maximum number of jobs. '
'Global explorers, object prepare and object run are '
'supported. Without argument CPU count is used by default. '
@ -229,7 +244,8 @@ def get_parsers():
dest='hostfile', required=False)
parser['config_args'].add_argument(
'-p', '--parallel', nargs='?', metavar='HOST_MAX',
type=check_positive_int,
type=functools.partial(check_lower_bounded_int, lower_bound=1,
name="positive int"),
help=('Operate on multiple hosts in parallel for specified maximum '
'hosts at a time. Without argument CPU count is used by '
'default.'),

17
cdist/conf/type/__install_config/gencode-local
View file

@ -22,27 +22,10 @@ chroot="$(cat "$__object/parameter/chroot")"
remote_exec="$__type/files/remote/exec"
remote_copy="$__type/files/remote/copy"
cdist_args=""
case "$__cdist_loglevel" in
INFO)
cdist_args="-v"
;;
VERBOSE)
cdist_args="-vv"
;;
DEBUG)
cdist_args="-vvv"
;;
TRACE)
cdist_args="-vvvv"
;;
esac
cat << DONE
export __default_remote_exec="$__remote_exec"
export __default_remote_copy="$__remote_copy"
cdist config \
$cdist_args \
--remote-exec="$remote_exec $chroot" \
--remote-copy="$remote_copy $chroot" \
$__target_host

2
cdist/conf/type/__install_stage/gencode-remote
View file

@ -22,7 +22,7 @@ uri="$(cat "$__object/parameter/uri" 2>/dev/null \
|| echo "$__object_id")"
target="$(cat "$__object/parameter/target")"
case "$__cdist_loglevel" in
case "$__cdist_log_level" in
DEBUG|TRACE)
curl="curl"
tar="tar -xvzp"

1
cdist/configuration.py
View file

@ -278,6 +278,7 @@ class Configuration(metaclass=Singleton):
'CDIST_REMOTE_COPY': 'remote_copy',
'CDIST_INVENTORY_DIR': 'inventory_dir',
'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern',
'__cdist_log_level': 'verbosity',
}
ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', )

1
cdist/core/__init__.py
View file

@ -28,3 +28,4 @@ from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code
from cdist.core.util import listdir
from cdist.core.util import log_level_env_var_val

2
cdist/core/code.py
View file

@ -22,6 +22,7 @@
#
import os
from . import util
'''
@ -107,6 +108,7 @@ class Code(object):
'__global': self.local.base_path,
'__files': self.local.files_path,
'__target_host_tags': self.local.target_host_tags,
'__cdist_log_level': util.log_level_env_var_val(local.log),
}
def _run_gencode(self, cdist_object, which):

2
cdist/core/explorer.py
View file

@ -25,6 +25,7 @@ import os
import glob
import multiprocessing
from cdist.mputil import mp_pool_run
from . import util
'''
common:
@ -78,6 +79,7 @@ class Explorer(object):
'__target_fqdn': self.target_host[2],
'__explorer': self.remote.global_explorer_path,
'__target_host_tags': self.local.target_host_tags,
'__cdist_log_level': util.log_level_env_var_val(self.log),
}
self._type_explorers_transferred = []
self.jobs = jobs

6
cdist/core/manifest.py
View file

@ -24,6 +24,7 @@ import logging
import os
import cdist
from . import util
'''
common:
@ -111,12 +112,9 @@ class Manifest(object):
'__target_fqdn': self.target_host[2],
'__files': self.local.files_path,
'__target_host_tags': self.local.target_host_tags,
'__cdist_log_level': util.log_level_env_var_val(self.log),
}
self.env.update(
{'__cdist_loglevel': logging.getLevelName(
self.log.getEffectiveLevel())})
def _open_logger(self):
self.log = logging.getLogger(self.target_host[0])

5
cdist/core/util.py
View file

@ -20,6 +20,7 @@
#
import os
import logging
def listdir(path='.', include_dot=False):
@ -34,3 +35,7 @@ def listdir(path='.', include_dot=False):
def _ishidden(path):
return path[0] in ('.', b'.'[0])
def log_level_env_var_val(log):
return logging.getLevelName(log.getEffectiveLevel())

6
cdist/emulator.py
View file

@ -109,9 +109,9 @@ class Emulator(object):
def __init_log(self):
"""Setup logging facility"""
if '__cdist_loglevel' in self.env:
if '__cdist_log_level' in self.env:
try:
loglevel = self.env['__cdist_loglevel']
loglevel = self.env['__cdist_log_level']
# For a text level it returns its numerical value.
level = logging.getLevelName(loglevel)
except ValueError:
@ -121,7 +121,7 @@ class Emulator(object):
try:
logging.root.setLevel(level)
except (ValueError, TypeError):
# if invalid __cdist_loglevel value
# if invalid __cdist_log_level value
logging.root.setLevel(logging.WARNING)
self.log = logging.getLogger(self.target_host[0])

3
cdist/test/code/__init__.py
View file

@ -23,6 +23,7 @@
import getpass
import os
import shutil
import logging
import cdist
from cdist import core
@ -100,6 +101,7 @@ class CodeTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__files'], self.local.files_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
self.assertEqual(output_dict['__cdist_log_level'], 'WARNING')
def test_run_gencode_remote_environment(self):
output_string = self.code.run_gencode_remote(self.cdist_object)
@ -125,6 +127,7 @@ class CodeTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__files'], self.local.files_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
self.assertEqual(output_dict['__cdist_log_level'], 'WARNING')
def test_transfer_code_remote(self):
self.cdist_object.code_remote = self.code.run_gencode_remote(

1
cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local
View file

@ -10,3 +10,4 @@ echo "echo __object_id: $__object_id"
echo "echo __object_name: $__object_name"
echo "echo __files: $__files"
echo "echo __target_host_tags: $__target_host_tags"
echo "echo __cdist_log_level: $__cdist_log_level"

12
cdist/test/emulator/__init__.py
View file

@ -64,8 +64,8 @@ class EmulatorTestCase(test.CdistTestCase):
self.manifest = core.Manifest(self.target_host, self.local)
self.env = self.manifest.env_initial_manifest(self.script)
self.env['__cdist_object_marker'] = self.local.object_marker_name
if '__cdist_loglevel' in self.env:
del self.env['__cdist_loglevel']
if '__cdist_log_level' in self.env:
del self.env['__cdist_log_level']
def tearDown(self):
shutil.rmtree(self.temp_dir)
@ -124,11 +124,11 @@ class EmulatorTestCase(test.CdistTestCase):
emu = emulator.Emulator(argv, env=self.env)
emu_loglevel = emu.log.getEffectiveLevel()
self.assertEqual(emu_loglevel, logging.WARNING)
self.env['__cdist_loglevel'] = logging.getLevelName(logging.DEBUG)
self.env['__cdist_log_level'] = logging.getLevelName(logging.DEBUG)
emu = emulator.Emulator(argv, env=self.env)
emu_loglevel = emu.log.getEffectiveLevel()
self.assertEqual(emu_loglevel, logging.DEBUG)
del self.env['__cdist_loglevel']
del self.env['__cdist_log_level']
def test_invalid_loglevel_value(self):
argv = ['__file', '/tmp/foobar']
@ -137,11 +137,11 @@ class EmulatorTestCase(test.CdistTestCase):
emu_loglevel = emu.log.getEffectiveLevel()
self.assertEqual(emu_loglevel, logging.WARNING)
# lowercase is invalid
self.env['__cdist_loglevel'] = 'debug'
self.env['__cdist_log_level'] = 'debug'
emu = emulator.Emulator(argv, env=self.env)
emu_loglevel = emu.log.getEffectiveLevel()
self.assertEqual(emu_loglevel, logging.WARNING)
del self.env['__cdist_loglevel']
del self.env['__cdist_log_level']
def test_requirement_via_order_dependency(self):
self.env['CDIST_ORDER_DEPENDENCY'] = 'on'

25
cdist/test/explorer/__init__.py
View file

@ -210,6 +210,31 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.assertEqual(names, output)
shutil.rmtree(out_path)
def test_explorer_environment(self):
cdist_type = core.CdistType(self.local.type_path, '__dump_env')
cdist_object = core.CdistObject(cdist_type, self.local.object_path,
self.local.object_marker_name,
'whatever')
self.explorer.transfer_type_explorers(cdist_type)
output = self.explorer.run_type_explorer('dump', cdist_object)
output_dict = {}
for line in output.split('\n'):
if line:
key, value = line.split(': ')
output_dict[key] = value
self.assertEqual(output_dict['__target_host'],
self.local.target_host[0])
self.assertEqual(output_dict['__target_hostname'],
self.local.target_host[1])
self.assertEqual(output_dict['__target_fqdn'],
self.local.target_host[2])
self.assertEqual(output_dict['__explorer'],
self.remote.global_explorer_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
self.assertEqual(output_dict['__cdist_log_level'], 'WARNING')
if __name__ == '__main__':
import unittest

8
cdist/test/explorer/fixtures/conf/type/__dump_env/explorer/dump Executable file
View file

@ -0,0 +1,8 @@
#!/bin/sh
echo "__target_host: $__target_host"
echo "__target_hostname: $__target_hostname"
echo "__target_fqdn: $__target_fqdn"
echo "__explorer: $__explorer"
echo "__target_host_tags: $__target_host_tags"
echo "__cdist_log_level: $__cdist_log_level"

20
cdist/test/manifest/__init__.py
View file

@ -73,7 +73,10 @@ class ManifestTestCase(test.CdistTestCase):
handle, output_file = self.mkstemp(dir=self.temp_dir)
os.close(handle)
os.environ['__cdist_test_out'] = output_file
self.manifest.run_initial_manifest(initial_manifest)
old_loglevel = logging.root.getEffectiveLevel()
self.log.setLevel(logging.VERBOSE)
manifest = cdist.core.manifest.Manifest(self.target_host, self.local)
manifest.run_initial_manifest(initial_manifest)
with open(output_file, 'r') as fd:
output_string = fd.read()
@ -96,6 +99,8 @@ class ManifestTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__files'], self.local.files_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
self.assertEqual(output_dict['__cdist_log_level'], 'VERBOSE')
self.log.setLevel(old_loglevel)
def test_type_manifest_environment(self):
cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
@ -105,7 +110,10 @@ class ManifestTestCase(test.CdistTestCase):
handle, output_file = self.mkstemp(dir=self.temp_dir)
os.close(handle)
os.environ['__cdist_test_out'] = output_file
self.manifest.run_type_manifest(cdist_object)
old_loglevel = self.log.getEffectiveLevel()
self.log.setLevel(logging.VERBOSE)
manifest = cdist.core.manifest.Manifest(self.target_host, self.local)
manifest.run_type_manifest(cdist_object)
with open(output_file, 'r') as fd:
output_string = fd.read()
@ -131,13 +139,15 @@ class ManifestTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__files'], self.local.files_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
self.assertEqual(output_dict['__cdist_log_level'], 'VERBOSE')
self.log.setLevel(old_loglevel)
def test_debug_env_setup(self):
def test_loglevel_env_setup(self):
current_level = self.log.getEffectiveLevel()
self.log.setLevel(logging.DEBUG)
manifest = cdist.core.manifest.Manifest(self.target_host, self.local)
self.assertTrue("__cdist_loglevel" in manifest.env)
self.assertEqual(manifest.env["__cdist_loglevel"], 'DEBUG')
self.assertTrue("__cdist_log_level" in manifest.env)
self.assertEqual(manifest.env["__cdist_log_level"], 'DEBUG')
self.log.setLevel(current_level)

1
cdist/test/manifest/fixtures/conf/manifest/dump_environment
View file

@ -10,4 +10,5 @@ __cdist_type_base_path: $__cdist_type_base_path
__manifest: $__manifest
__files: $__files
__target_host_tags: $__target_host_tags
__cdist_log_level: $__cdist_log_level
DONE

1
cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest
View file

@ -14,4 +14,5 @@ __object_id: $__object_id
__object_name: $__object_name
__files: $__files
__target_host_tags: $__target_host_tags
__cdist_log_level: $__cdist_log_level
DONE

2
docs/changelog
View file

@ -13,6 +13,8 @@ next:
* Documentation: Document __cdist_loglevel type variable (Darko Poljak)
* Type __install_stage: Fix __debug -> __cdist_loglevel (Darko Poljak)
* Core, types: Make __cdist_loglevel value more expressive (Darko Poljak)
* Core: Add -l/--log-level option (Darko Poljak)
* Core: __cdist_loglevel -> __cdist_log_level and make cdist honor __cdist_log_level env var (Darko Poljak)
4.6.1: 2017-08-30
* Type __user: Explore with /etc files (passwd, group, shadow) (Philippe Gregoire)

12
docs/src/cdist-reference.rst.sh
View file

@ -198,10 +198,11 @@ Environment variables (for reading)
-----------------------------------
The following environment variables are exported by cdist:
__cdist_loglevel
__cdist_log_level
String value of cdist log level. One of OFF, ERROR, WARNING, INFO,
VERBOSE, DEBUG and TRACE.
Available for: initial manifest, type manifest, type gencode.
Available for: initial manifest, explorer, type manifest, type explorer,
type gencode.
__explorer
Directory that contains all global explorers.
Available for: initial manifest, explorer, type explorer, shell.
@ -264,6 +265,13 @@ The following environment variables influence the behaviour of cdist:
require
Setup dependencies between objects (see \`cdist manifest <cdist-manifest.html>\`_).
__cdist_log_level
String value of cdist log level. One of OFF, ERROR, WARNING, INFO,
VERBOSE, DEBUG and TRACE. If set cdist will set this log level in
accordance with configuration rules. If cdist invokation is used
in types then nested cdist will honor this specified log level if
not specified otherwise while invoking it.
CDIST_PATH
Colon delimited list of config directories.

5
docs/src/cdist-type.rst
View file

@ -333,9 +333,10 @@ So when you generate a script with the following content, it will work:
Log level in types
------------------
cdist log level can be accessed from __cdist_loglevel variable.
cdist log level can be accessed from __cdist_log_level variable.
Value is a string, one of OFF, ERROR, WARNING, INFO, VERBOSE, DEBUG and
TRACE. It is available for initial manifest, type manifest and type gencode.
TRACE. It is available for initial manifest, explorer, type manifest,
type explorer, type gencode.
Hints for typewriters

58
docs/src/man1/cdist.rst
View file

@ -11,11 +11,12 @@ SYNOPSIS
::
cdist [-h] [-q] [-v] [-V] {banner,config,install,inventory,shell} ...
cdist [-h] [-l LOGLEVEL] [-q] [-v] [-V]
{banner,config,install,inventory,shell} ...
cdist banner [-h] [-q] [-v]
cdist banner [-h] [-l LOGLEVEL] [-q] [-v]
cdist config [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
[-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
[-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
@ -23,7 +24,7 @@ SYNOPSIS
[-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]]
cdist install [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
[-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
[-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
@ -31,32 +32,35 @@ SYNOPSIS
[-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]]
cdist inventory [-h] [-q] [-v] [-b] [-g CONFIG_FILE] [-I INVENTORY_DIR]
cdist inventory [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR]
{add-host,add-tag,del-host,del-tag,list} ...
cdist inventory add-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-f HOSTFILE]
cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
[-g CONFIG_FILE] [-I INVENTORY_DIR]
[-f HOSTFILE]
[host [host ...]]
cdist inventory add-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
[-t TAGLIST]
cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
[-g CONFIG_FILE] [-I INVENTORY_DIR]
[-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
[host [host ...]]
cdist inventory del-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE]
cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b]
[-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE]
[host [host ...]]
cdist inventory del-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE]
[-T TAGFILE] [-t TAGLIST]
cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b]
[-g CONFIG_FILE] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
[host [host ...]]
cdist inventory list [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
[host [host ...]]
cdist shell [-h] [-q] [-v] [-s SHELL]
cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL]
DESCRIPTION
@ -75,16 +79,28 @@ All commands accept the following options:
Show the help screen.
.. option:: -l LOGLEVEL, --log-level LOGLEVEL
Set the specified verbosity level. The levels, in
order from the lowest to the highest, are: ERROR (-1),
WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4
or higher). If used along with -v then -v increases
last set value and -l overwrites last set value.
.. option:: -q, --quiet
Quiet mode: disables logging, including WARNING and ERROR.
.. option:: -v, --verbose
Increase the verbosity level. Every instance of -v increments the verbosity
level by one. Its default value is 0 which includes ERROR and WARNING levels.
The levels, in order from the lowest to the highest, are:
ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4 or higher).
Increase the verbosity level. Every instance of -v
increments the verbosity level by one. Its default
value is 0 which includes ERROR and WARNING levels.
The levels, in order from the lowest to the highest,
are: ERROR (-1), WARNING (0), INFO (1), VERBOSE (2),
DEBUG (3) TRACE (4 or higher). If used along with -l
then -l overwrites last set value and -v increases
last set value.
.. option:: -V, --version

3
scripts/cdist
View file

@ -22,6 +22,7 @@
#
import logging
import sys
import cdist
import cdist.argparse
import cdist.banner
@ -56,8 +57,6 @@ def commandline():
if __name__ == "__main__":
import sys
cdistpythonversion = '3.2'
if sys.version < cdistpythonversion:
print('Python >= {} is required on the source host.'.format(