From bdee7273af7d4960ca85511bb460737fc504cad8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 30 Aug 2017 23:02:17 +0200 Subject: [PATCH] Configfile (#559) Add cdist configuration/config file support. --- cdist/argparse.py | 49 +- cdist/config.py | 26 +- cdist/configuration.py | 421 ++++++++ cdist/exec/local.py | 28 +- cdist/exec/remote.py | 12 +- cdist/inventory.py | 67 +- cdist/test/configuration/__init__.py | 1076 +++++++++++++++++++ cdist/test/configuration/fixtures/.nonempty | 0 cdist/test/exec/local.py | 6 + docs/src/man1/cdist.rst | 223 ++-- scripts/cdist | 28 +- 11 files changed, 1800 insertions(+), 136 deletions(-) create mode 100644 cdist/configuration.py create mode 100644 cdist/test/configuration/__init__.py create mode 100644 cdist/test/configuration/fixtures/.nonempty diff --git a/cdist/argparse.py b/cdist/argparse.py index 4501774c..f0ea802a 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -1,9 +1,9 @@ import argparse import cdist import multiprocessing -import os import logging import collections +import cdist.configuration # set of beta sub-commands @@ -115,7 +115,7 @@ def get_parsers(): '-b', '--beta', help=('Enable beta functionality. '), action='store_true', dest='beta', - default='CDIST_BETA' in os.environ) + default=False) # Main subcommand parser parser['main'] = argparse.ArgumentParser( @@ -136,12 +136,18 @@ def get_parsers(): '-I', '--inventory', help=('Use specified custom inventory directory. ' 'Inventory directory is set up by the following rules: ' - 'if this argument is set then specified directory is used, ' - 'if CDIST_INVENTORY_DIR env var is set then its value is ' - 'used, if HOME env var is set then ~/.cdist/inventory is ' + 'if cdist configuration resolves this value then specified ' + 'directory is used, ' + 'if HOME env var is set then ~/.cdist/inventory is ' 'used, otherwise distribution inventory directory is used.'), dest="inventory_dir", required=False) + parser['common'] = argparse.ArgumentParser(add_help=False) + parser['common'].add_argument( + '-g', '--config-file', + help=('Use specified custom configuration file.'), + dest="config_file", required=False) + # Config parser['config_main'] = argparse.ArgumentParser(add_help=False) parser['config_main'].add_argument( @@ -149,7 +155,7 @@ def get_parsers(): help=('Specify custom cache path pattern. If ' 'it is not set then default hostdir is used.'), dest='cache_path_pattern', - default=os.environ.get('CDIST_CACHE_PATH_PATTERN')) + default=None) parser['config_main'].add_argument( '-c', '--conf-dir', help=('Add configuration directory (can be repeated, ' @@ -195,13 +201,13 @@ def get_parsers(): '--remote-copy', help='Command to use for remote copy (should behave like scp).', action='store', dest='remote_copy', - default=os.environ.get('CDIST_REMOTE_COPY')) + default=None) parser['config_main'].add_argument( '--remote-exec', help=('Command to use for remote execution ' '(should behave like ssh).'), action='store', dest='remote_exec', - default=os.environ.get('CDIST_REMOTE_EXEC')) + default=None) # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) @@ -243,6 +249,7 @@ def get_parsers(): dest='tag', required=False, action="store_true", default=False) parser['config'] = parser['sub'].add_parser( 'config', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['config_main'], parser['inventory_common'], parser['config_args']]) @@ -256,12 +263,14 @@ def get_parsers(): # Inventory parser['inventory'] = parser['sub'].add_parser( 'inventory', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['inventory_common']]) parser['invsub'] = parser['inventory'].add_subparsers( title="Inventory commands", dest="subcommand") parser['add-host'] = parser['invsub'].add_parser( 'add-host', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['inventory_common']]) parser['add-host'].add_argument( 'host', nargs='*', help='Host(s) to add.') @@ -275,6 +284,7 @@ def get_parsers(): parser['add-tag'] = parser['invsub'].add_parser( 'add-tag', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['inventory_common']]) parser['add-tag'].add_argument( 'host', nargs='*', @@ -305,6 +315,7 @@ def get_parsers(): parser['del-host'] = parser['invsub'].add_parser( 'del-host', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['inventory_common']]) parser['del-host'].add_argument( 'host', nargs='*', help='Host(s) to delete.') @@ -321,6 +332,7 @@ def get_parsers(): parser['del-tag'] = parser['invsub'].add_parser( 'del-tag', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['inventory_common']]) parser['del-tag'].add_argument( 'host', nargs='*', @@ -355,6 +367,7 @@ def get_parsers(): parser['list'] = parser['invsub'].add_parser( 'list', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['inventory_common']]) parser['list'].add_argument( 'host', nargs='*', help='Host(s) to list.') @@ -401,3 +414,23 @@ def handle_loglevel(args): args.verbose = _verbosity_level_off logging.root.setLevel(_verbosity_level[args.verbose]) + + +def parse_and_configure(argv): + parser = get_parsers() + parser_args = parser['main'].parse_args(argv) + cfg = cdist.configuration.Configuration(parser_args) + args = cfg.get_args() + # Loglevels are handled globally in here + handle_loglevel(args) + + log = logging.getLogger("cdist") + + log.verbose("version %s" % cdist.VERSION) + log.trace('command line args: {}'.format(cfg.command_line_args)) + log.trace('configuration: {}'.format(cfg.get_config())) + log.trace('configured args: {}'.format(args)) + + check_beta(vars(args)) + + return parser, cfg diff --git a/cdist/config.py b/cdist/config.py index 3e786272..779365f3 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -31,15 +31,12 @@ import multiprocessing from cdist.mputil import mp_pool_run, mp_sig_handler import atexit import shutil - import cdist import cdist.hostsource - import cdist.exec.local import cdist.exec.remote - import cdist.util.ipaddr as ipaddr - +import cdist.configuration from cdist import core, inventory from cdist.util.remoteutil import inspect_ssh_mux_opts @@ -162,8 +159,11 @@ class Config(object): hostcnt = 0 + cfg = cdist.configuration.Configuration(args) + configuration = cfg.get_config(section='GLOBAL') + if args.tag or args.all_tagged_hosts: - inventory.determine_default_inventory_dir(args) + inventory.determine_default_inventory_dir(args, configuration) if args.all_tagged_hosts: inv_list = inventory.InventoryList( hosts=None, istag=True, hostfile=None, @@ -191,7 +191,7 @@ class Config(object): else: # if configuring by host then check inventory for tags host = entry - inventory.determine_default_inventory_dir(args) + inventory.determine_default_inventory_dir(args, configuration) inv_list = inventory.InventoryList( hosts=(host,), db_basedir=args.inventory_dir) inv = tuple(inv_list.entries()) @@ -208,14 +208,16 @@ class Config(object): hostcnt += 1 if args.parallel: - pargs = (host, host_tags, host_base_path, hostdir, args, True) + pargs = (host, host_tags, host_base_path, hostdir, args, True, + configuration) log.trace(("Args for multiprocessing operation " "for host {}: {}".format(host, pargs))) process_args.append(pargs) else: try: cls.onehost(host, host_tags, host_base_path, hostdir, - args, parallel=False) + args, parallel=False, + configuration=configuration) except cdist.Error as e: failed_hosts.append(host) if args.parallel and len(process_args) == 1: @@ -281,7 +283,7 @@ class Config(object): @classmethod def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, - parallel): + parallel, configuration): """Configure ONE system. If operating in parallel then return tuple (host, True|False, ) so that main process knows for which host function was successful. @@ -308,7 +310,8 @@ class Config(object): initial_manifest=args.manifest, add_conf_dirs=args.conf_dir, cache_path_pattern=args.cache_path_pattern, - quiet_mode=args.quiet) + quiet_mode=args.quiet, + configuration=configuration) remote = cdist.exec.remote.Remote( target_host=target_host, @@ -316,7 +319,8 @@ class Config(object): remote_copy=remote_copy, base_path=args.remote_out_path, quiet_mode=args.quiet, - archiving_mode=args.use_archiving) + archiving_mode=args.use_archiving, + configuration=configuration) cleanup_cmds = [] if cleanup_cmd: diff --git a/cdist/configuration.py b/cdist/configuration.py new file mode 100644 index 00000000..fa9671d0 --- /dev/null +++ b/cdist/configuration.py @@ -0,0 +1,421 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import configparser +import os +import cdist +import cdist.argparse +import re +import multiprocessing + + +class Singleton(type): + instance = None + + def __call__(cls, *args, **kwargs): + if not cls.instance: + cls.instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls.instance + + +_VERBOSITY_VALUES = ( + 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE', 'OFF', +) +_ARCHIVING_VALUES = ( + 'tar', 'tgz', 'tbz2', 'txz', 'none', +) + + +class OptionBase: + def __init__(self, name): + self.name = name + + def converter(self, *args, **kwargs): + raise NotImplementedError('Subclass should implement this method') + + def translate(self, val): + return val + + def update_value(self, currval, newval, update_appends=False): + '''Update current option value currval with new option value newval. + If update_appends is True and if currval and newval are lists then + resulting list contains all values in currval plus all values in + newval. Otherwise, newval is returned. + ''' + if (isinstance(currval, list) and isinstance(newval, list) and + update_appends): + rv = [] + if currval: + rv.extend(currval) + if newval: + rv.extend(newval) + if not rv: + rv = None + return rv + else: + return newval + + +class StringOption(OptionBase): + def __init__(self, name): + super().__init__(name) + + def converter(self): + def string_converter(val): + return self.translate(str(val)) + return string_converter + + def translate(self, val): + if val: + return val + else: + return None + + +class BooleanOption(OptionBase): + BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES + + def __init__(self, name): + super().__init__(name) + + def converter(self): + def boolean_converter(val): + v = val.lower() + if v not in self.BOOLEAN_STATES: + raise ValueError('Invalid {} boolean value: {}'.format( + self.name, val)) + return self.translate(v) + return boolean_converter + + def translate(self, val): + return self.BOOLEAN_STATES[val] + + +class IntOption(OptionBase): + def __init__(self, name): + super().__init__(name) + + def converter(self): + def int_converter(val): + return self.translate(int(val)) + return int_converter + + +class LowerBoundIntOption(IntOption): + def __init__(self, name, lower_bound): + super().__init__(name) + self.lower_bound = lower_bound + + def converter(self): + def lower_bound_converter(val): + converted = super(LowerBoundIntOption, self).converter()(val) + if converted < self.lower_bound: + raise ValueError("Invalid {} value: {} < {}".format( + self.name, val, self.lower_bound)) + return converted + return lower_bound_converter + + +class SpecialCasesLowerBoundIntOption(LowerBoundIntOption): + def __init__(self, name, lower_bound, special_cases_mapping): + super().__init__(name, lower_bound) + self.special_cases_mapping = special_cases_mapping + + def translate(self, val): + if val in self.special_cases_mapping: + return self.special_cases_mapping[val] + else: + return val + + +class JobsOption(SpecialCasesLowerBoundIntOption): + def __init__(self, name): + super().__init__(name, -1, {-1: multiprocessing.cpu_count()}) + + +class SelectOption(OptionBase): + def __init__(self, name, valid_values): + super().__init__(name) + self.valid_values = valid_values + + def converter(self): + def select_converter(val): + if val in self.valid_values: + return self.translate(val) + else: + raise ValueError("Invalid {} value: {}.".format( + self.name, val)) + return select_converter + + +class VerbosityOption(SelectOption): + def __init__(self): + super().__init__('verbosity', _VERBOSITY_VALUES) + + def translate(self, val): + name = 'VERBOSE_' + val + verbose = getattr(cdist.argparse, name) + return verbose + + +class DelimitedValuesOption(OptionBase): + def __init__(self, name, delimiter): + super().__init__(name) + self.delimiter = delimiter + + def converter(self): + def delimited_values_converter(val): + vals = re.split(r'(?. +# +# + +import configparser +import os +import multiprocessing +import cdist.configuration as cc +import os.path as op +import argparse +from cdist import test +import cdist.argparse as cap + + +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') + + +class ConfigurationOptionsTestCase(test.CdistTestCase): + + def test_OptionBase(self): + option = cc.OptionBase('test') + test_cases = ( + ([], [], True, None, ), + (['spam', 'eggs', ], [], True, ['spam', 'eggs', ], ), + ([], ['spam', 'eggs', ], True, ['spam', 'eggs', ], ), + ( + ['spam', 'eggs', ], + ['ham', 'spamspam', ], + True, + ['spam', 'eggs', 'ham', 'spamspam', ], + ), + (['spam', 'eggs', ], 'spam:eggs', True, 'spam:eggs', ), + ('spam:eggs', ['spam', 'eggs', ], True, ['spam', 'eggs', ], ), + ('spam', 'eggs', True, 'eggs', ), + + (['spam', 'eggs', ], 'spam:eggs', True, 'spam:eggs', ), + + ('spam:eggs', ['spam', 'eggs', ], False, ['spam', 'eggs', ], ), + ('spam', 'eggs', False, 'eggs', ), + ( + ['spam', 'eggs', ], + ['ham', 'spamspam', ], + False, + ['ham', 'spamspam', ], + ), + ) + for currval, newval, update_appends, expected in test_cases: + self.assertEqual(option.update_value(currval, newval, + update_appends=update_appends), + expected) + + def test_StringOption(self): + option = cc.StringOption('test') + self.assertIsNone(option.translate('')) + self.assertEqual(option.translate('spam'), 'spam') + converter = option.converter() + self.assertEqual(converter('spam'), 'spam') + self.assertIsNone(converter('')) + + def test_BooleanOption(self): + option = cc.BooleanOption('test') + for x in cc.BooleanOption.BOOLEAN_STATES: + self.assertEqual(option.translate(x), + cc.BooleanOption.BOOLEAN_STATES[x]) + converter = option.converter() + self.assertRaises(ValueError, converter, 'of') + for x in cc.BooleanOption.BOOLEAN_STATES: + self.assertEqual(converter(x), cc.BooleanOption.BOOLEAN_STATES[x]) + + def test_IntOption(self): + option = cc.IntOption('test') + converter = option.converter() + self.assertRaises(ValueError, converter, 'x') + for x in range(-5, 10): + self.assertEqual(converter(str(x)), x) + + def test_LowerBoundIntOption(self): + option = cc.LowerBoundIntOption('test', -1) + converter = option.converter() + self.assertRaises(ValueError, converter, -2) + for x in range(-1, 10): + self.assertEqual(converter(str(x)), x) + + def test_SpecialCasesLowerBoundIntOption(self): + special_cases = { + -1: 8, + -2: 10, + } + option = cc.SpecialCasesLowerBoundIntOption('test', -1, special_cases) + for x in special_cases: + self.assertEqual(option.translate(x), special_cases[x]) + + def test_SelectOption(self): + valid_values = ('spam', 'eggs', 'ham', ) + option = cc.SelectOption('test', valid_values) + converter = option.converter() + self.assertRaises(ValueError, converter, 'spamspam') + for x in valid_values: + self.assertEqual(converter(x), x) + + def test_DelimitedValuesOption(self): + option = cc.DelimitedValuesOption('test', ':') + converter = option.converter() + value = 'spam:eggs::ham' + self.assertEqual(converter(value), ['spam', 'eggs', 'ham', ]) + self.assertIsNone(converter('')) + + +class ConfigurationTestCase(test.CdistTestCase): + + def setUp(self): + # Create test config file. + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + config_custom = configparser.ConfigParser() + config_custom['GLOBAL'] = { + 'parallel': '4', + 'archiving': 'txz', + } + + config_custom2 = configparser.ConfigParser() + config_custom2['GLOBAL'] = { + 'parallel': '16', + 'archiving': 'tbz2', + 'remote_copy': 'myscp', + } + + self.expected_config_dict = { + 'GLOBAL': { + 'beta': False, + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': None, + 'cache_path_pattern': None, + 'conf_dir': None, + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': None, + 'remote_exec': None, + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': None, + }, + } + + self.config_file = os.path.join(fixtures, 'cdist.cfg') + with open(self.config_file, 'w') as f: + config.write(f) + + self.custom_config_file = os.path.join(fixtures, 'cdist_custom.cfg') + with open(self.custom_config_file, 'w') as f: + config_custom.write(f) + + self.custom_config_file2 = os.path.join(fixtures, 'cdist_custom2.cfg') + with open(self.custom_config_file2, 'w') as f: + config_custom2.write(f) + + config['TEST'] = {} + self.invalid_config_file1 = os.path.join(fixtures, + 'cdist_invalid1.cfg') + with open(self.invalid_config_file1, 'w') as f: + config.write(f) + + del config['TEST'] + config['GLOBAL']['test'] = 'test' + self.invalid_config_file2 = os.path.join(fixtures, + 'cdist_invalid2.cfg') + with open(self.invalid_config_file2, 'w') as f: + config.write(f) + + del config['GLOBAL']['test'] + config['GLOBAL']['archiving'] = 'zip' + self.invalid_config_file3 = os.path.join(fixtures, + 'cdist_invalid3.cfg') + with open(self.invalid_config_file3, 'w') as f: + config.write(f) + + self.maxDiff = None + + def tearDown(self): + os.remove(self.config_file) + os.remove(self.custom_config_file) + os.remove(self.custom_config_file2) + os.remove(self.invalid_config_file1) + os.remove(self.invalid_config_file2) + os.remove(self.invalid_config_file3) + + # remove files from tests + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + if os.path.exists(global_config_file): + os.remove(global_config_file) + if os.path.exists(local_config_file): + os.remove(local_config_file) + if os.path.exists(custom_config_file): + os.remove(custom_config_file) + + def test_singleton(self): + x = cc.Configuration(None) + args = argparse.Namespace() + args.a = 'a' + y = cc.Configuration() + self.assertIs(x, y) + + def test_read_config_file(self): + config = cc.Configuration(None, env={}, config_files=()) + d = config._read_config_file(self.config_file) + self.assertEqual(d, self.expected_config_dict) + + for x in range(1, 4): + config_file = getattr(self, 'invalid_config_file' + str(x)) + with self.assertRaises(ValueError): + config._read_config_file(config_file) + + def test_read_env_var_config(self): + config = cc.Configuration(None, env={}, config_files=()) + env = { + 'a': 'a', + 'CDIST_BETA': '1', + 'CDIST_PATH': '/usr/local/cdist:~/.cdist', + } + expected = { + 'beta': True, + 'conf_dir': ['/usr/local/cdist', '~/.cdist', ], + } + section = 'GLOBAL' + d = config._read_env_var_config(env, section) + self.assertEqual(d, expected) + + del env['CDIST_BETA'] + del expected['beta'] + d = config._read_env_var_config(env, section) + self.assertEqual(d, expected) + + def test_read_args_config(self): + config = cc.Configuration(None, env={}, config_files=()) + args = argparse.Namespace() + args.beta = False + args.conf_dir = ['/usr/local/cdist1', ] + args.verbose = 3 + args.tag = 'test' + + expected = { + 'beta': False, + 'conf_dir': ['/usr/local/cdist1', ], + 'verbosity': 3, + } + args_dict = vars(args) + d = config._read_args_config(args_dict) + self.assertEqual(d, expected) + self.assertNotEqual(d, args_dict) + + def test_update_config_dict(self): + config = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', ], + 'parallel': -1, + }, + } + newconfig = { + 'GLOBAL': { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + expected = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', '~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration = cc.Configuration(None, env={}, config_files=()) + configuration._update_config_dict(config, newconfig, + update_appends=True) + self.assertEqual(config, expected) + expected = { + 'GLOBAL': { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration._update_config_dict(config, newconfig, + update_appends=False) + self.assertEqual(config, expected) + + def test_update_config_dict_section(self): + config = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', ], + 'parallel': -1, + }, + } + newconfig = { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + } + expected = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', '~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration = cc.Configuration(None, env={}, config_files=()) + configuration._update_config_dict_section('GLOBAL', config, newconfig, + update_appends=True) + self.assertEqual(config, expected) + expected = { + 'GLOBAL': { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration._update_config_dict_section('GLOBAL', config, newconfig, + update_appends=False) + self.assertEqual(config, expected) + + def test_configuration1(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + } + args = argparse.Namespace() + expected_config_dict = { + 'GLOBAL': { + }, + } + + # bypass singleton so we can test further + cc.Configuration.instance = None + configuration = cc.Configuration(args, env=env) + self.assertIsNotNone(configuration.args) + self.assertIsNotNone(configuration.env) + self.assertIsNotNone(configuration.config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration2(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': False, + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': None, + 'cache_path_pattern': None, + 'conf_dir': None, + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': None, + 'remote_exec': None, + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': None, + }, + } + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration3(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': ['/opt/cdist', ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'tar', + }, + } + config_files = (global_config_file, local_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration4(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': None, + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/cdist/conf', + '/usr/local/share/cdist/conf', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': None, + 'remote_exec': None, + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': None, + }, + } + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration5(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/cdist/conf', + '/usr/local/share/cdist/conf', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'tar', + }, + } + config_files = (global_config_file, local_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration6(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + args.inventory_dir = '/opt/sysadmin/cdist/inventory' + args.conf_dir = ['/opt/sysadmin/cdist/conf', ] + args.manifest = '/opt/sysadmin/cdist/conf/manifest/init' + args.jobs = 10 + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/opt/sysadmin/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/cdist/conf', + '/usr/local/share/cdist/conf', + '/opt/sysadmin/cdist/conf', + ], + 'init_manifest': '/opt/sysadmin/cdist/conf/manifest/init', + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'jobs': 10, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'tar', + }, + } + config_files = (global_config_file, local_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration7(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'conf_dir': '/opt/conf/cdist', + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'parallel': '15', + 'archiving': 'txz', + } + + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + with open(custom_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'txz', + }, + } + + config_files = (global_config_file, local_config_file, ) + + args.config_file = custom_config_file + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration8(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'conf_dir': '/opt/conf/cdist', + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'parallel': '15', + 'archiving': 'txz', + } + + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + with open(custom_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'txz', + }, + } + + config_files = (global_config_file, local_config_file, ) + + os.environ['CDIST_CONFIG_FILE'] = custom_config_file + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_get_args(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + config = configparser.ConfigParser() + config['GLOBAL'] = { + 'conf_dir': '/opt/conf/cdist', + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'parallel': '15', + 'archiving': 'txz', + } + + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + with open(custom_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'txz', + }, + } + + config_files = (global_config_file, local_config_file, ) + + os.environ['CDIST_CONFIG_FILE'] = custom_config_file + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + args = configuration.get_args() + dargs = vars(args) + expected_args = { + 'beta': True, + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbose': cap.VERBOSE_INFO, + 'use_archiving': 'txz', + } + + self.assertEqual(dargs, expected_args) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/cdist/test/configuration/fixtures/.nonempty b/cdist/test/configuration/fixtures/.nonempty new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/exec/local.py b/cdist/test/exec/local.py index 6336947c..1b298143 100644 --- a/cdist/test/exec/local.py +++ b/cdist/test/exec/local.py @@ -28,8 +28,10 @@ import string import random import time import datetime +import argparse import cdist +import cdist.configuration as cc from cdist import test from cdist.exec import local @@ -165,6 +167,9 @@ class LocalTestCase(test.CdistTestCase): os.environ['CDIST_PATH'] = conf_dir + configuration = cc.Configuration(argparse.Namespace(), + env=os.environ) + link_test_local = local.Local( target_host=( 'localhost', @@ -174,6 +179,7 @@ class LocalTestCase(test.CdistTestCase): base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=test.cdist_exec_path, + configuration=configuration ) link_test_local._create_conf_path_and_link_conf_dirs() diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 79485594..6ffa0f98 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -15,43 +15,45 @@ SYNOPSIS cdist banner [-h] [-q] [-v] - cdist config [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] - [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] - [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] - [-s] [-t] + cdist config [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a] + [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist install [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] - [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] - [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]] - [-s] [-t] + cdist install [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]] + [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a] + [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist inventory [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] + cdist inventory [-h] [-q] [-v] [-b] [-g CONFIG_FILE] [-I INVENTORY_DIR] {add-host,add-tag,del-host,del-tag,list} ... - cdist inventory add-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] - [-f HOSTFILE] + cdist inventory add-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-f HOSTFILE] [host [host ...]] - cdist inventory add-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] - [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + cdist inventory add-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE] + [-t TAGLIST] [host [host ...]] - cdist inventory del-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] + cdist inventory del-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [host [host ...]] - cdist inventory del-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + cdist inventory del-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] + [-T TAGFILE] [-t TAGLIST] [host [host ...]] - cdist inventory list [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a] - [-f HOSTFILE] [-H] [-t] + cdist inventory list [-h] [-q] [-v] [-b] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] [host [host ...]] cdist shell [-h] [-q] [-v] [-s SHELL] @@ -112,7 +114,7 @@ Install command is currently in beta. .. option:: -b, --beta Enable beta functionality. - + .. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN Sepcify custom cache path pattern. If it is not set then @@ -133,15 +135,18 @@ Install command is currently in beta. read hosts from stdin. For the file format see :strong:`HOSTFILE FORMAT` below. +.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE + + Use specified custom configuration file. + .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR Use specified custom inventory directory. Inventory - directory is set up by the following rules: if this - argument is set then specified directory is used, if - CDIST_INVENTORY_DIR env var is set then its value is - used, if HOME env var is set then ~/.cdit/inventory is - used, otherwise distribution inventory directory is - used. + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. .. option:: -i MANIFEST, --initial-manifest MANIFEST @@ -254,16 +259,18 @@ Add host(s) to inventory database. host or host file is specified then, by default, read from stdin. Hostfile format is the same as config hostfile format. +.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE + + Use specified custom configuration file. .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR Use specified custom inventory directory. Inventory - directory is set up by the following rules: if this - argument is set then specified directory is used, if - CDIST_INVENTORY_DIR env var is set then its value is - used, if HOME env var is set then ~/.cdist/inventory is - used, otherwise distribution inventory directory is - used. + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. INVENTORY ADD-TAG @@ -287,15 +294,18 @@ Add tag(s) to inventory database. are specified then tags are read from stdin and are added to all hosts. Hostfile format is the same as config hostfile format. +.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE + + Use specified custom configuration file. + .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR Use specified custom inventory directory. Inventory - directory is set up by the following rules: if this - argument is set then specified directory is used, if - CDIST_INVENTORY_DIR env var is set then its value is - used, if HOME env var is set then ~/.cdist/inventory is - used, otherwise distribution inventory directory is - used. + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. .. option:: -T TAGFILE, --tag-file TAGFILE @@ -335,15 +345,18 @@ Delete host(s) from inventory database. host or host file is specified then, by default, read from stdin. Hostfile format is the same as config hostfile format. +.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE + + Use specified custom configuration file. + .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR Use specified custom inventory directory. Inventory - directory is set up by the following rules: if this - argument is set then specified directory is used, if - CDIST_INVENTORY_DIR env var is set then its value is - used, if HOME env var is set then ~/.cdist/inventory is - used, otherwise distribution inventory directory is - used. + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. INVENTORY DEL-TAG @@ -372,15 +385,18 @@ Delete tag(s) from inventory database. from stdin and are deleted from all hosts. Hostfile format is the same as config hostfile format. +.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE + + Use specified custom configuration file. + .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR Use specified custom inventory directory. Inventory - directory is set up by the following rules: if this - argument is set then specified directory is used, if - CDIST_INVENTORY_DIR env var is set then its value is - used, if HOME env var is set then ~/.cdist/inventory is - used, otherwise distribution inventory directory is - used. + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. .. option:: -T TAGFILE, --tag-file TAGFILE @@ -421,6 +437,10 @@ List inventory database. host or host file is specified then, by default, list all. Hostfile format is the same as config hostfile format. +.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE + + Use specified custom configuration file. + .. option:: -H, --host-only Suppress tags listing. @@ -428,12 +448,11 @@ List inventory database. .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR Use specified custom inventory directory. Inventory - directory is set up by the following rules: if this - argument is set then specified directory is used, if - CDIST_INVENTORY_DIR env var is set then its value is - used, if HOME env var is set then ~/.cdist/inventory is - used, otherwise distribution inventory directory is - used. + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. .. option:: -t, --tag @@ -454,6 +473,82 @@ usage. Its primary use is for debugging type parameters. be POSIX compatible shell. +CONFIGURATION FILE +------------------ +cdist obtains configuration data from the following sources in the following +order: + + #. command-line options + #. configuration file specified at command-line + #. configuration file specified in CDIST_CONFIG_FILE environment variable + #. environment variables + #. user's configuration file (first one found of ~/.cdist.cfg, + $XDG_CONFIG_HOME/cdist/cdist.cfg, in specified order) + #. system-wide configuration file (/etc/cdist.cfg) + +if one exists. + +cdist configuration file is in the INI file format. Currently it supports +only [GLOBAL] section. +The possible keywords and their meanings are as follows: + +:strong:`archiving` + Use specified archiving. Valid values include: + 'none', 'tar', 'tgz', 'tbz2' and 'txz'. + +:strong:`beta` + Enable beta functionality. It recognizes boolean values from + 'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0' + +:strong:`cache_path_pattern` + Specify cache path pattern. + +:strong:`conf_dir` + Comma separated list of configuration directories. + If also specified at command line then values from command line are + appended to this value. + +:strong:`init_manifest` + Specify default initial manifest. + +:strong:`inventory_dir` + Specify inventory directory. + +:strong:`jobs` + Specify number of jobs for parallel processing. If -1 then the default, + number of CPU's in the system is used. If 0 then parallel processing in + jobs is disabled. If set to positive number then specified maximum + number of processes will be used. + +:strong:`local_shell` + Shell command used for local execution. + +:strong:`out_path` + Directory to save cdist output in. + +:strong:`parallel` + Process hosts in parallel. If -1 then the default, number of CPU's in + the system is used. If 0 then parallel processing of hosts is disabled. + If set to positive number then specified maximum number of processes + will be used. + +:strong:`remote_copy` + Command to use for remote copy (should behave like scp). + +:strong:`remote_exec` + Command to use for remote execution (should behave like ssh). + +:strong:`remote_out_path` + Directory to save cdist output in on the target host. + +:strong:`remote_shell` + Shell command at remote host used for remote execution. + +:strong:`verbosity` + Set verbosity level. Valid values are: + 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. + + FILES ----- ~/.cdist @@ -468,6 +563,10 @@ cdist/conf cdist/inventory The distribution inventory directory. This path is relative to cdist installation directory. +/etc/cdist.cfg + Global cdist configuration file, if exists. +~/.cdist/cdist.cfg + Local cdist configuration file, if exists. NOTES ----- @@ -592,6 +691,10 @@ CDIST_BETA CDIST_CACHE_PATH_PATTERN Custom cache path pattern. +CDIST_CONFIG_FILE + Custom configuration file. + + EXIT STATUS ----------- The following exit values shall be returned: diff --git a/scripts/cdist b/scripts/cdist index 384610b1..93fa392d 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -22,26 +22,20 @@ # import logging +import cdist +import cdist.argparse +import cdist.banner +import cdist.config +import cdist.install +import cdist.shell +import cdist.inventory def commandline(): """Parse command line""" - import cdist.argparse - import cdist.banner - import cdist.config - import cdist.install - import cdist.shell - import cdist.inventory - - parser = cdist.argparse.get_parsers() - args = parser['main'].parse_args(sys.argv[1:]) - - # Loglevels are handled globally in here - cdist.argparse.handle_loglevel(args) - - log.verbose("version %s" % cdist.VERSION) - log.trace(args) + parser, cfg = cdist.argparse.parse_and_configure(sys.argv[1:]) + args = cfg.get_args() # Work around python 3.3 bug: # http://bugs.python.org/issue16308 @@ -58,7 +52,6 @@ def commandline(): parser['main'].print_help() sys.exit(0) - cdist.argparse.check_beta(vars(args)) args.func(args) @@ -74,9 +67,8 @@ if __name__ == "__main__": exit_code = 0 try: - import os import re - import cdist + import os log = logging.getLogger("cdist")