Configfile (#559)

Add cdist configuration/config file support.

cdist/argparse.py

@@ -1,9 +1,9 @@
 import argparse
 import cdist
 import multiprocessing
-import os
 import logging
 import collections
+import cdist.configuration
 
 
 # set of beta sub-commands
@@ -115,7 +115,7 @@ def get_parsers():
            '-b', '--beta',
            help=('Enable beta functionality. '),
            action='store_true', dest='beta',
-           default='CDIST_BETA' in os.environ)
+           default=False)
 
     # Main subcommand parser
     parser['main'] = argparse.ArgumentParser(
@@ -136,12 +136,18 @@ def get_parsers():
            '-I', '--inventory',
            help=('Use specified custom inventory directory. '
                  'Inventory directory is set up by the following rules: '
-                 'if this argument is set then specified directory is used, '
+                 'if cdist configuration resolves this value then specified '
-                 'if CDIST_INVENTORY_DIR env var is set then its value is '
+                 'directory is used, '
-                 'used, if HOME env var is set then ~/.cdist/inventory is '
+                 'if HOME env var is set then ~/.cdist/inventory is '
                  'used, otherwise distribution inventory directory is used.'),
            dest="inventory_dir", required=False)
 
+    parser['common'] = argparse.ArgumentParser(add_help=False)
+    parser['common'].add_argument(
+           '-g', '--config-file',
+           help=('Use specified custom configuration file.'),
+           dest="config_file", required=False)
+
     # Config
     parser['config_main'] = argparse.ArgumentParser(add_help=False)
     parser['config_main'].add_argument(
@@ -149,7 +155,7 @@ def get_parsers():
             help=('Specify custom cache path pattern. If '
                   'it is not set then default hostdir is used.'),
             dest='cache_path_pattern',
-            default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
+            default=None)
     parser['config_main'].add_argument(
             '-c', '--conf-dir',
             help=('Add configuration directory (can be repeated, '
@@ -195,13 +201,13 @@ def get_parsers():
            '--remote-copy',
            help='Command to use for remote copy (should behave like scp).',
            action='store', dest='remote_copy',
-           default=os.environ.get('CDIST_REMOTE_COPY'))
+           default=None)
     parser['config_main'].add_argument(
            '--remote-exec',
            help=('Command to use for remote execution '
                  '(should behave like ssh).'),
            action='store', dest='remote_exec',
-           default=os.environ.get('CDIST_REMOTE_EXEC'))
+           default=None)
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
@@ -243,6 +249,7 @@ def get_parsers():
              dest='tag', required=False, action="store_true", default=False)
     parser['config'] = parser['sub'].add_parser(
             'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['common'],
                                parser['config_main'],
                                parser['inventory_common'],
                                parser['config_args']])
@@ -256,12 +263,14 @@ def get_parsers():
     # Inventory
     parser['inventory'] = parser['sub'].add_parser(
            'inventory', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
                                  parser['inventory_common']])
     parser['invsub'] = parser['inventory'].add_subparsers(
             title="Inventory commands", dest="subcommand")
 
     parser['add-host'] = parser['invsub'].add_parser(
             'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
                                  parser['inventory_common']])
     parser['add-host'].add_argument(
             'host', nargs='*', help='Host(s) to add.')
@@ -275,6 +284,7 @@ def get_parsers():
 
     parser['add-tag'] = parser['invsub'].add_parser(
             'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['common'],
                                 parser['inventory_common']])
     parser['add-tag'].add_argument(
            'host', nargs='*',
@@ -305,6 +315,7 @@ def get_parsers():
 
     parser['del-host'] = parser['invsub'].add_parser(
             'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['common'],
                                  parser['inventory_common']])
     parser['del-host'].add_argument(
             'host', nargs='*', help='Host(s) to delete.')
@@ -321,6 +332,7 @@ def get_parsers():
 
     parser['del-tag'] = parser['invsub'].add_parser(
             'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['common'],
                                 parser['inventory_common']])
     parser['del-tag'].add_argument(
             'host', nargs='*',
@@ -355,6 +367,7 @@ def get_parsers():
 
     parser['list'] = parser['invsub'].add_parser(
             'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['common'],
                              parser['inventory_common']])
     parser['list'].add_argument(
             'host', nargs='*', help='Host(s) to list.')
@@ -401,3 +414,23 @@ def handle_loglevel(args):
         args.verbose = _verbosity_level_off
 
     logging.root.setLevel(_verbosity_level[args.verbose])
+
+
+def parse_and_configure(argv):
+    parser = get_parsers()
+    parser_args = parser['main'].parse_args(argv)
+    cfg = cdist.configuration.Configuration(parser_args)
+    args = cfg.get_args()
+    # Loglevels are handled globally in here
+    handle_loglevel(args)
+
+    log = logging.getLogger("cdist")
+
+    log.verbose("version %s" % cdist.VERSION)
+    log.trace('command line args: {}'.format(cfg.command_line_args))
+    log.trace('configuration: {}'.format(cfg.get_config()))
+    log.trace('configured args: {}'.format(args))
+
+    check_beta(vars(args))
+
+    return parser, cfg

cdist/config.py

@@ -31,15 +31,12 @@ import multiprocessing
 from cdist.mputil import mp_pool_run, mp_sig_handler
 import atexit
 import shutil
-
 import cdist
 import cdist.hostsource
-
 import cdist.exec.local
 import cdist.exec.remote
-
 import cdist.util.ipaddr as ipaddr
-
+import cdist.configuration
 from cdist import core, inventory
 from cdist.util.remoteutil import inspect_ssh_mux_opts
 
@@ -162,8 +159,11 @@ class Config(object):
 
         hostcnt = 0
 
+        cfg = cdist.configuration.Configuration(args)
+        configuration = cfg.get_config(section='GLOBAL')
+
         if args.tag or args.all_tagged_hosts:
-            inventory.determine_default_inventory_dir(args)
+            inventory.determine_default_inventory_dir(args, configuration)
             if args.all_tagged_hosts:
                 inv_list = inventory.InventoryList(
                     hosts=None, istag=True, hostfile=None,
@@ -191,7 +191,7 @@ class Config(object):
             else:
                 # if configuring by host then check inventory for tags
                 host = entry
-                inventory.determine_default_inventory_dir(args)
+                inventory.determine_default_inventory_dir(args, configuration)
                 inv_list = inventory.InventoryList(
                     hosts=(host,), db_basedir=args.inventory_dir)
                 inv = tuple(inv_list.entries())
@@ -208,14 +208,16 @@ class Config(object):
 
             hostcnt += 1
             if args.parallel:
-                pargs = (host, host_tags, host_base_path, hostdir, args, True)
+                pargs = (host, host_tags, host_base_path, hostdir, args, True,
+                         configuration)
                 log.trace(("Args for multiprocessing operation "
                            "for host {}: {}".format(host, pargs)))
                 process_args.append(pargs)
             else:
                 try:
                     cls.onehost(host, host_tags, host_base_path, hostdir,
-                                args, parallel=False)
+                                args, parallel=False,
+                                configuration=configuration)
                 except cdist.Error as e:
                     failed_hosts.append(host)
         if args.parallel and len(process_args) == 1:
@@ -281,7 +283,7 @@ class Config(object):
 
     @classmethod
     def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
-                parallel):
+                parallel, configuration):
         """Configure ONE system.
            If operating in parallel then return tuple (host, True|False, )
            so that main process knows for which host function was successful.
@@ -308,7 +310,8 @@ class Config(object):
                 initial_manifest=args.manifest,
                 add_conf_dirs=args.conf_dir,
                 cache_path_pattern=args.cache_path_pattern,
-                quiet_mode=args.quiet)
+                quiet_mode=args.quiet,
+                configuration=configuration)
 
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
@@ -316,7 +319,8 @@ class Config(object):
                 remote_copy=remote_copy,
                 base_path=args.remote_out_path,
                 quiet_mode=args.quiet,
-                archiving_mode=args.use_archiving)
+                archiving_mode=args.use_archiving,
+                configuration=configuration)
 
             cleanup_cmds = []
             if cleanup_cmd:

cdist/configuration.py

@@ -0,0 +1,421 @@
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+import configparser
+import os
+import cdist
+import cdist.argparse
+import re
+import multiprocessing
+
+
+class Singleton(type):
+    instance = None
+
+    def __call__(cls, *args, **kwargs):
+        if not cls.instance:
+            cls.instance = super(Singleton, cls).__call__(*args, **kwargs)
+        return cls.instance
+
+
+_VERBOSITY_VALUES = (
+    'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE', 'OFF',
+)
+_ARCHIVING_VALUES = (
+    'tar', 'tgz', 'tbz2', 'txz', 'none',
+)
+
+
+class OptionBase:
+    def __init__(self, name):
+        self.name = name
+
+    def converter(self, *args, **kwargs):
+        raise NotImplementedError('Subclass should implement this method')
+
+    def translate(self, val):
+        return val
+
+    def update_value(self, currval, newval, update_appends=False):
+        '''Update current option value currval with new option value newval.
+        If update_appends is True and if currval and newval are lists then
+        resulting list contains all values in currval plus all values in
+        newval. Otherwise, newval is returned.
+        '''
+        if (isinstance(currval, list) and isinstance(newval, list) and
+                update_appends):
+            rv = []
+            if currval:
+                rv.extend(currval)
+            if newval:
+                rv.extend(newval)
+            if not rv:
+                rv = None
+            return rv
+        else:
+            return newval
+
+
+class StringOption(OptionBase):
+    def __init__(self, name):
+        super().__init__(name)
+
+    def converter(self):
+        def string_converter(val):
+            return self.translate(str(val))
+        return string_converter
+
+    def translate(self, val):
+        if val:
+            return val
+        else:
+            return None
+
+
+class BooleanOption(OptionBase):
+    BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES
+
+    def __init__(self, name):
+        super().__init__(name)
+
+    def converter(self):
+        def boolean_converter(val):
+            v = val.lower()
+            if v not in self.BOOLEAN_STATES:
+                raise ValueError('Invalid {} boolean value: {}'.format(
+                    self.name, val))
+            return self.translate(v)
+        return boolean_converter
+
+    def translate(self, val):
+        return self.BOOLEAN_STATES[val]
+
+
+class IntOption(OptionBase):
+    def __init__(self, name):
+        super().__init__(name)
+
+    def converter(self):
+        def int_converter(val):
+            return self.translate(int(val))
+        return int_converter
+
+
+class LowerBoundIntOption(IntOption):
+    def __init__(self, name, lower_bound):
+        super().__init__(name)
+        self.lower_bound = lower_bound
+
+    def converter(self):
+        def lower_bound_converter(val):
+            converted = super(LowerBoundIntOption, self).converter()(val)
+            if converted < self.lower_bound:
+                raise ValueError("Invalid {} value: {} < {}".format(
+                    self.name, val, self.lower_bound))
+            return converted
+        return lower_bound_converter
+
+
+class SpecialCasesLowerBoundIntOption(LowerBoundIntOption):
+    def __init__(self, name, lower_bound, special_cases_mapping):
+        super().__init__(name, lower_bound)
+        self.special_cases_mapping = special_cases_mapping
+
+    def translate(self, val):
+        if val in self.special_cases_mapping:
+            return self.special_cases_mapping[val]
+        else:
+            return val
+
+
+class JobsOption(SpecialCasesLowerBoundIntOption):
+    def __init__(self, name):
+        super().__init__(name, -1, {-1: multiprocessing.cpu_count()})
+
+
+class SelectOption(OptionBase):
+    def __init__(self, name, valid_values):
+        super().__init__(name)
+        self.valid_values = valid_values
+
+    def converter(self):
+        def select_converter(val):
+            if val in self.valid_values:
+                return self.translate(val)
+            else:
+                raise ValueError("Invalid {} value: {}.".format(
+                    self.name, val))
+        return select_converter
+
+
+class VerbosityOption(SelectOption):
+    def __init__(self):
+        super().__init__('verbosity', _VERBOSITY_VALUES)
+
+    def translate(self, val):
+        name = 'VERBOSE_' + val
+        verbose = getattr(cdist.argparse, name)
+        return verbose
+
+
+class DelimitedValuesOption(OptionBase):
+    def __init__(self, name, delimiter):
+        super().__init__(name)
+        self.delimiter = delimiter
+
+    def converter(self):
+        def delimited_values_converter(val):
+            vals = re.split(r'(?<!\\)' + self.delimiter, val)
+            vals = [x for x in vals if x]
+            return self.translate(vals)
+        return delimited_values_converter
+
+    def translate(self, val):
+        if val:
+            return val
+        else:
+            return None
+
+
+class ConfDirOption(DelimitedValuesOption):
+    def __init__(self):
+        super().__init__('conf_dir', os.pathsep)
+
+
+class ArchivingOption(SelectOption):
+    def __init__(self):
+        super().__init__('archiving', _ARCHIVING_VALUES)
+
+    def translate(self, val):
+        if val == 'none':
+            return None
+        else:
+            return val
+
+
+_ARG_OPTION_MAPPING = {
+    'beta': 'beta',
+    'cache_path_pattern': 'cache_path_pattern',
+    'conf_dir': 'conf_dir',
+    'manifest': 'init_manifest',
+    'out_path': 'out_path',
+    'remote_out_path': 'remote_out_path',
+    'remote_copy': 'remote_copy',
+    'remote_exec': 'remote_exec',
+    'inventory_dir': 'inventory_dir',
+    'jobs': 'jobs',
+    'parallel': 'parallel',
+    'verbose': 'verbosity',
+    'use_archiving': 'archiving',
+}
+
+
+class Configuration(metaclass=Singleton):
+    _config_basename = 'cdist.cfg'
+    _global_config_file = os.path.join('/', 'etc', _config_basename, )
+    _local_config_file = os.path.join(os.path.expanduser('~'),
+                                      '.' + _config_basename, )
+    if (not (os.path.exists(_local_config_file) and
+             os.path.isfile(_local_config_file))):
+        _local_config_file = os.path.join(
+            os.environ.get('XDG_CONFIG_HOME',
+                           os.path.expanduser('~/.config/cdist')),
+            _config_basename)
+    default_config_files = (_global_config_file, _local_config_file, )
+    ENV_VAR_CONFIG_FILE = 'CDIST_CONFIG_FILE'
+
+    VERBOSITY_VALUES = _VERBOSITY_VALUES
+    ARCHIVING_VALUES = _ARCHIVING_VALUES
+
+    CONFIG_FILE_OPTIONS = {
+        'GLOBAL': {
+            'beta': BooleanOption('beta'),
+            'local_shell': StringOption('local_shell'),
+            'remote_shell': StringOption('remote_shell'),
+            'cache_path_pattern': StringOption('cache_path_pattern'),
+            'conf_dir': ConfDirOption(),
+            'init_manifest': StringOption('init_manifest'),
+            'out_path': StringOption('out_path'),
+            'remote_out_path': StringOption('remote_out_path'),
+            'remote_copy': StringOption('remote_copy'),
+            'remote_exec': StringOption('remote_exec'),
+            'inventory_dir': StringOption('inventory_dir'),
+            'jobs': JobsOption('jobs'),
+            'parallel': JobsOption('parallel'),
+            'verbosity': VerbosityOption(),
+            'archiving': ArchivingOption(),
+        },
+    }
+
+    ENV_VAR_OPTION_MAPPING = {
+        'CDIST_BETA': 'beta',
+        'CDIST_PATH': 'conf_dir',
+        'CDIST_LOCAL_SHELL': 'local_shell',
+        'CDIST_REMOTE_SHELL': 'remote_shell',
+        'CDIST_REMOTE_EXEC': 'remote_exec',
+        'CDIST_REMOTE_COPY': 'remote_copy',
+        'CDIST_INVENTORY_DIR': 'inventory_dir',
+        'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern',
+    }
+    ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', )
+
+    ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING
+    ADJUST_ARG_OPTION_MAPPING = {
+       _ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING
+    }
+
+    def _convert_args(self, args):
+        if args:
+            if hasattr(args, '__dict__'):
+                return vars(args)
+            else:
+                raise ValueError(
+                    'args parameter must be have __dict__ attribute')
+        else:
+            return None
+
+    def __init__(self, command_line_args, env=os.environ,
+                 config_files=default_config_files):
+        self.command_line_args = command_line_args
+        self.args = self._convert_args(command_line_args)
+        self.env = env
+        self.config_files = config_files
+        self.config = self._get_config()
+
+    def get_config(self, section=None):
+        if section is None:
+            return self.config
+        if section in self.config:
+            return self.config[section]
+        raise ValueError('Unknown section: {}'.format(section))
+
+    def _get_args_name_value(self, arg_name, val):
+        if arg_name == 'verbosity' and val == 'OFF':
+            name = 'quiet'
+            rv = True
+        else:
+            name = arg_name
+            rv = val
+        return (name, rv)
+
+    def get_args(self, section='GLOBAL'):
+        args = self.command_line_args
+        cfg = self.get_config(section)
+        for option in self.ADJUST_ARG_OPTION_MAPPING:
+            if option in cfg:
+                arg_name = self.ADJUST_ARG_OPTION_MAPPING[option]
+                val = cfg[option]
+                name, val = self._get_args_name_value(arg_name, val)
+                setattr(args, name, val)
+        return args
+
+    def _read_config_file(self, files):
+        config_parser = configparser.ConfigParser()
+        config_parser.read(files)
+        d = dict()
+        for section in config_parser.sections():
+            if section not in self.CONFIG_FILE_OPTIONS:
+                raise ValueError("Invalid section: {}.".format(section))
+            if section not in d:
+                d[section] = dict()
+
+            for option in config_parser[section]:
+                if option not in self.CONFIG_FILE_OPTIONS[section]:
+                    raise ValueError("Invalid option: {}.".format(option))
+
+                option_object = self.CONFIG_FILE_OPTIONS[section][option]
+                converter = option_object.converter()
+                val = config_parser[section][option]
+                newval = converter(val)
+                d[section][option] = newval
+        return d
+
+    def _read_env_var_config(self, env, section):
+        d = dict()
+        for option in self.ENV_VAR_OPTION_MAPPING:
+            if option in env:
+                dst_opt = self.ENV_VAR_OPTION_MAPPING[option]
+                if option in self.ENV_VAR_BOOLEAN_OPTIONS:
+                    d[dst_opt] = True
+                else:
+                    option_object = self.CONFIG_FILE_OPTIONS[section][dst_opt]
+                    converter = option_object.converter()
+                    val = env[option]
+                    newval = converter(val)
+                    d[dst_opt] = newval
+        return d
+
+    def _read_args_config(self, args):
+        d = dict()
+        for option in self.ARG_OPTION_MAPPING:
+            if option in args:
+                dst_opt = self.ARG_OPTION_MAPPING[option]
+                d[dst_opt] = args[option]
+        return d
+
+    def _update_config_dict(self, config, newconfig, update_appends=False):
+        for section in newconfig:
+            self._update_config_dict_section(
+                section, config, newconfig[section], update_appends)
+
+    def _update_config_dict_section(self, section, config, newconfig,
+                                    update_appends=False):
+        if section not in config:
+            config[section] = dict()
+        for option in newconfig:
+            newval = newconfig[option]
+            if option in config[section]:
+                currval = config[section][option]
+            else:
+                currval = None
+            option_object = self.CONFIG_FILE_OPTIONS[section][option]
+            config[section][option] = option_object.update_value(
+                currval, newval, update_appends)
+
+    def _get_config(self):
+        # global config file
+        # local config file
+        config = self._read_config_file(self.config_files)
+        # default empty config if needed
+        if not config:
+            config['GLOBAL'] = dict()
+        # environment variables
+        newconfig = self._read_env_var_config(self.env, 'GLOBAL')
+        for section in config:
+            self._update_config_dict_section(section, config, newconfig)
+        # config file in CDIST_CONFIG_FILE env var
+        config_file = os.environ.get(self.ENV_VAR_CONFIG_FILE, None)
+        if config_file:
+            newconfig = self._read_config_file(config_file)
+            self._update_config_dict(config, newconfig)
+        # command line config file
+        if (self.args and 'config_file' in self.args and
+                self.args['config_file']):
+            newconfig = self._read_config_file(self.args['config_file'])
+            self._update_config_dict(config, newconfig)
+        # command line
+        if self.args:
+            newconfig = self._read_args_config(self.args)
+            for section in config:
+                self._update_config_dict_section(section, config, newconfig,
+                                                 update_appends=True)
+        return config

cdist/exec/local.py

@@ -55,7 +55,8 @@ class Local(object):
                  initial_manifest=None,
                  add_conf_dirs=None,
                  cache_path_pattern=None,
-                 quiet_mode=False):
+                 quiet_mode=False,
+                 configuration=None):
 
         self.target_host = target_host
         if target_host_tags is None:
@@ -70,6 +71,10 @@ class Local(object):
         self._add_conf_dirs = add_conf_dirs
         self.cache_path_pattern = cache_path_pattern
         self.quiet_mode = quiet_mode
+        if configuration:
+            self.configuration = configuration
+        else:
+            self.configuration = {}
 
         self._init_log()
         self._init_permissions()
@@ -142,10 +147,12 @@ class Local(object):
             self.conf_dirs.append(self.home_dir)
 
         # Add directories defined in the CDIST_PATH environment variable
-        if 'CDIST_PATH' in os.environ:
+        # if 'CDIST_PATH' in os.environ:
-            cdist_path_dirs = re.split(r'(?<!\\):', os.environ['CDIST_PATH'])
+        #     cdist_path_dirs = re.split(r'(?<!\\):', os.environ['CDIST_PATH'])
-            cdist_path_dirs.reverse()
+        #     cdist_path_dirs.reverse()
-            self.conf_dirs.extend(cdist_path_dirs)
+        #     self.conf_dirs.extend(cdist_path_dirs)
+        if 'conf_dir' in self.configuration:
+            self.conf_dirs.extend(self.configuration['conf_dir'])
 
         # Add command line supplied directories
         if self._add_conf_dirs:
@@ -175,12 +182,11 @@ class Local(object):
     def _init_cache_dir(self, cache_dir):
         if cache_dir:
             self.cache_path = cache_dir
+        elif self.home_dir:
+            self.cache_path = os.path.join(self.home_dir, "cache")
         else:
-            if self.home_dir:
+            raise cdist.Error(
-                self.cache_path = os.path.join(self.home_dir, "cache")
+                "No homedir setup and no cache dir location given")
-            else:
-                raise cdist.Error(
-                        "No homedir setup and no cache dir location given")
 
     def rmdir(self, path):
         """Remove directory on the local side."""
@@ -258,7 +264,7 @@ class Local(object):
             self.log.debug('%s is executable, running it', script)
             command = [script]
         else:
-            command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
+            command = [self.configuration.get('local_shell', "/bin/sh"), "-e"]
             self.log.debug('%s is NOT executable, running it with %s',
                            script, " ".join(command))
             command.append(script)

cdist/exec/remote.py

@@ -62,7 +62,8 @@ class Remote(object):
                  remote_copy,
                  base_path=None,
                  quiet_mode=None,
-                 archiving_mode=None):
+                 archiving_mode=None,
+                 configuration=None):
         self.target_host = target_host
         self._exec = remote_exec
         self._copy = remote_copy
@@ -73,6 +74,10 @@ class Remote(object):
             self.base_path = "/var/lib/cdist"
         self.quiet_mode = quiet_mode
         self.archiving_mode = archiving_mode
+        if configuration:
+            self.configuration = configuration
+        else:
+            self.configuration = {}
 
         self.conf_path = os.path.join(self.base_path, "conf")
         self.object_path = os.path.join(self.base_path, "object")
@@ -235,7 +240,10 @@ class Remote(object):
 
         """
 
-        command = [os.environ.get('CDIST_REMOTE_SHELL', "/bin/sh"), "-e"]
+        command = [
+            self.configuration.get('CDIST_REMOTE_SHELL', "/bin/sh"),
+            "-e"
+        ]
         command.append(script)
 
         return self.run(command, env, return_output)

cdist/inventory.py

@@ -26,6 +26,7 @@ import os
 import os.path
 import itertools
 import sys
+import cdist.configuration
 from cdist.hostsource import hostfile_process_line
 
 DIST_INVENTORY_DB_NAME = "inventory"
@@ -34,21 +35,23 @@ dist_inventory_db = os.path.abspath(os.path.join(
     os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME))
 
 
-def determine_default_inventory_dir(args):
+def determine_default_inventory_dir(args, configuration):
     # The order of inventory dir setting by decreasing priority
-    # 1. inventory_dir argument
+    # 1. inventory_dir from configuration
-    # 2. CDIST_INVENTORY_DIR env var if set
+    # 2. ~/.cdist/inventory if HOME env var is set
-    # 3. ~/.cdist/inventory if HOME env var is set
+    # 3. distribution inventory directory
-    # 4. distribution inventory directory
+    inventory_dir_set = False
-    if not args.inventory_dir:
+    if 'inventory_dir' in configuration:
-        if 'CDIST_INVENTORY_DIR' in os.environ:
+        val = configuration['inventory_dir']
-            args.inventory_dir = os.environ['CDIST_INVENTORY_DIR']
+        if val:
+            args.inventory_dir = val
+            inventory_dir_set = True
+    if not inventory_dir_set:
+        home = cdist.home_dir()
+        if home:
+            args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
         else:
-            home = cdist.home_dir()
+            args.inventory_dir = dist_inventory_db
-            if home:
-                args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
-            else:
-                args.inventory_dir = dist_inventory_db
 
 
 def contains_all(big, little):
@@ -80,8 +83,12 @@ def rstrip_nl(s):
 class Inventory(object):
     """Inventory main class"""
 
-    def __init__(self, db_basedir=dist_inventory_db):
+    def __init__(self, db_basedir=dist_inventory_db, configuration=None):
         self.db_basedir = db_basedir
+        if configuration:
+            self.configuration = configuration
+        else:
+            self.configuration = {}
         self.log = logging.getLogger("inventory")
         self.init_db()
 
@@ -171,7 +178,10 @@ class Inventory(object):
         log = logging.getLogger("inventory")
         if 'taglist' in args:
             args.taglist = cls.strlist_to_list(args.taglist)
-        determine_default_inventory_dir(args)
+
+        cfg = cdist.configuration.Configuration(args)
+        configuration = cfg.get_config(section='GLOBAL')
+        determine_default_inventory_dir(args, configuration)
 
         log.debug("Using inventory: {}".format(args.inventory_dir))
         log.trace("Inventory args: {}".format(vars(args)))
@@ -182,23 +192,26 @@ class Inventory(object):
                               hostfile=args.hostfile,
                               db_basedir=args.inventory_dir,
                               list_only_host=args.list_only_host,
-                              has_all_tags=args.has_all_tags)
+                              has_all_tags=args.has_all_tags,
+                              configuration=configuration)
         elif args.subcommand == "add-host":
             c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
-                              db_basedir=args.inventory_dir)
+                              db_basedir=args.inventory_dir,
+                              configuration=configuration)
         elif args.subcommand == "del-host":
             c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
                               all=args.all, db_basedir=args.inventory_dir,
-                              action="del")
+                              action="del", configuration=configuration)
         elif args.subcommand == "add-tag":
             c = InventoryTag(hosts=args.host, tags=args.taglist,
                              hostfile=args.hostfile, tagfile=args.tagfile,
-                             db_basedir=args.inventory_dir)
+                             db_basedir=args.inventory_dir,
+                             configuration=configuration)
         elif args.subcommand == "del-tag":
             c = InventoryTag(hosts=args.host, tags=args.taglist,
                              hostfile=args.hostfile, tagfile=args.tagfile,
                              all=args.all, db_basedir=args.inventory_dir,
-                             action="del")
+                             action="del", configuration=configuration)
         else:
             raise cdist.Error("Unknown inventory command \'{}\'".format(
                         args.subcommand))
@@ -208,8 +221,8 @@ class Inventory(object):
 class InventoryList(Inventory):
     def __init__(self, hosts=None, istag=False, hostfile=None,
                  list_only_host=False, has_all_tags=False,
-                 db_basedir=dist_inventory_db):
+                 db_basedir=dist_inventory_db, configuration=None):
-        super().__init__(db_basedir)
+        super().__init__(db_basedir, configuration)
         self.hosts = hosts
         self.istag = istag
         self.hostfile = hostfile
@@ -274,8 +287,9 @@ class InventoryList(Inventory):
 
 class InventoryHost(Inventory):
     def __init__(self, hosts=None, hostfile=None,
-                 db_basedir=dist_inventory_db, all=False, action="add"):
+                 db_basedir=dist_inventory_db, all=False, action="add",
-        super().__init__(db_basedir)
+                 configuration=None):
+        super().__init__(db_basedir, configuration)
         self.actions = ("add", "del")
         if action not in self.actions:
             raise cdist.Error("Invalid action \'{}\', valid actions are:"
@@ -323,8 +337,9 @@ class InventoryHost(Inventory):
 
 class InventoryTag(Inventory):
     def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None,
-                 db_basedir=dist_inventory_db, all=False, action="add"):
+                 db_basedir=dist_inventory_db, all=False, action="add",
-        super().__init__(db_basedir)
+                 configuration=None):
+        super().__init__(db_basedir, configuration)
         self.actions = ("add", "del")
         if action not in self.actions:
             raise cdist.Error("Invalid action \'{}\', valid actions are:"

cdist/test/exec/local.py

@@ -28,8 +28,10 @@ import string
 import random
 import time
 import datetime
+import argparse
 
 import cdist
+import cdist.configuration as cc
 from cdist import test
 from cdist.exec import local
 
@@ -165,6 +167,9 @@ class LocalTestCase(test.CdistTestCase):
 
         os.environ['CDIST_PATH'] = conf_dir
 
+        configuration = cc.Configuration(argparse.Namespace(),
+                                         env=os.environ)
+
         link_test_local = local.Local(
             target_host=(
                 'localhost',
@@ -174,6 +179,7 @@ class LocalTestCase(test.CdistTestCase):
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=test.cdist_exec_path,
+            configuration=configuration
         )
 
         link_test_local._create_conf_path_and_link_conf_dirs()

docs/src/man1/cdist.rst

@@ -15,43 +15,45 @@ SYNOPSIS
 
     cdist banner [-h] [-q] [-v]
 
-    cdist config [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
+    cdist config [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                 [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
+                 [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
-                 [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR]
+                 [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
-                 [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                 [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
-                 [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]]
+                 [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
-                 [-s] [-t]
+                 [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
                  [host [host ...]] 
 
-    cdist install [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
+    cdist install [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                  [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
+                  [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
-                  [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR]
+                  [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
-                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                  [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
-                  [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]]
+                  [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
-                  [-s] [-t]
+                  [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
                   [host [host ...]] 
 
-    cdist inventory [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+    cdist inventory [-h] [-q] [-v] [-b] [-g CONFIG_FILE] [-I INVENTORY_DIR]
                     {add-host,add-tag,del-host,del-tag,list} ...
 
-    cdist inventory add-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+    cdist inventory add-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                             [-f HOSTFILE]
+                             [-I INVENTORY_DIR] [-f HOSTFILE]
                              [host [host ...]]
 
-    cdist inventory add-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+    cdist inventory add-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
+                            [-t TAGLIST]
                             [host [host ...]]
 
-    cdist inventory del-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+    cdist inventory del-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                             [-f HOSTFILE]
+                             [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
                              [host [host ...]]
 
-    cdist inventory del-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+    cdist inventory del-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [-I INVENTORY_DIR] [-a] [-f HOSTFILE]
+                            [-T TAGFILE] [-t TAGLIST]
                             [host [host ...]]
 
-    cdist inventory list [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+    cdist inventory list [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
-                         [-f HOSTFILE] [-H] [-t]
+                         [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
                          [host [host ...]]
 
     cdist shell [-h] [-q] [-v] [-s SHELL]
@@ -133,15 +135,18 @@ Install command is currently in beta.
     read hosts from stdin. For the file format see
     :strong:`HOSTFILE FORMAT` below.
 
+.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
+
+    Use specified custom configuration file.
+
 .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
 
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if this
+    directory is set up by the following rules: if cdist 
-    argument is set then specified directory is used, if
+    configuration resolves this value then specified
-    CDIST_INVENTORY_DIR env var is set then its value is
+    directory is used, if HOME env var is set then
-    used, if HOME env var is set then ~/.cdit/inventory is
+    ~/.cdit/inventory is used, otherwise distribution
-    used, otherwise distribution inventory directory is
+    inventory directory is used.
-    used.
 
 .. option:: -i MANIFEST, --initial-manifest MANIFEST
 
@@ -254,16 +259,18 @@ Add host(s) to inventory database.
     host or host file is specified then, by default, read
     from stdin. Hostfile format is the same as config hostfile format.
 
+.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
+
+    Use specified custom configuration file.
 
 .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
 
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if this
+    directory is set up by the following rules: if cdist 
-    argument is set then specified directory is used, if
+    configuration resolves this value then specified
-    CDIST_INVENTORY_DIR env var is set then its value is
+    directory is used, if HOME env var is set then
-    used, if HOME env var is set then ~/.cdist/inventory is
+    ~/.cdit/inventory is used, otherwise distribution
-    used, otherwise distribution inventory directory is
+    inventory directory is used.
-    used.
 
 
 INVENTORY ADD-TAG
@@ -287,15 +294,18 @@ Add tag(s) to inventory database.
     are specified then tags are read from stdin and are
     added to all hosts. Hostfile format is the same as config hostfile format.
 
+.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
+
+    Use specified custom configuration file.
+
 .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
 
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if this
+    directory is set up by the following rules: if cdist 
-    argument is set then specified directory is used, if
+    configuration resolves this value then specified
-    CDIST_INVENTORY_DIR env var is set then its value is
+    directory is used, if HOME env var is set then
-    used, if HOME env var is set then ~/.cdist/inventory is
+    ~/.cdit/inventory is used, otherwise distribution
-    used, otherwise distribution inventory directory is
+    inventory directory is used.
-    used.
 
 .. option:: -T TAGFILE, --tag-file TAGFILE
 
@@ -335,15 +345,18 @@ Delete host(s) from inventory database.
     host or host file is specified then, by default, read
     from stdin. Hostfile format is the same as config hostfile format.
 
+.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
+
+    Use specified custom configuration file.
+
 .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
 
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if this
+    directory is set up by the following rules: if cdist 
-    argument is set then specified directory is used, if
+    configuration resolves this value then specified
-    CDIST_INVENTORY_DIR env var is set then its value is
+    directory is used, if HOME env var is set then
-    used, if HOME env var is set then ~/.cdist/inventory is
+    ~/.cdit/inventory is used, otherwise distribution
-    used, otherwise distribution inventory directory is
+    inventory directory is used.
-    used.
 
 
 INVENTORY DEL-TAG
@@ -372,15 +385,18 @@ Delete tag(s) from inventory database.
     from stdin and are deleted from all hosts. Hostfile
     format is the same as config hostfile format.
 
+.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
+
+    Use specified custom configuration file.
+
 .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
 
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if this
+    directory is set up by the following rules: if cdist 
-    argument is set then specified directory is used, if
+    configuration resolves this value then specified
-    CDIST_INVENTORY_DIR env var is set then its value is
+    directory is used, if HOME env var is set then
-    used, if HOME env var is set then ~/.cdist/inventory is
+    ~/.cdit/inventory is used, otherwise distribution
-    used, otherwise distribution inventory directory is
+    inventory directory is used.
-    used.
 
 .. option:: -T TAGFILE, --tag-file TAGFILE
 
@@ -421,6 +437,10 @@ List inventory database.
     host or host file is specified then, by default, list
     all. Hostfile format is the same as config hostfile format.
 
+.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
+
+    Use specified custom configuration file.
+
 .. option:: -H, --host-only
 
     Suppress tags listing.
@@ -428,12 +448,11 @@ List inventory database.
 .. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
 
     Use specified custom inventory directory. Inventory
-    directory is set up by the following rules: if this
+    directory is set up by the following rules: if cdist 
-    argument is set then specified directory is used, if
+    configuration resolves this value then specified
-    CDIST_INVENTORY_DIR env var is set then its value is
+    directory is used, if HOME env var is set then
-    used, if HOME env var is set then ~/.cdist/inventory is
+    ~/.cdit/inventory is used, otherwise distribution
-    used, otherwise distribution inventory directory is
+    inventory directory is used.
-    used.
 
 .. option:: -t, --tag
 
@@ -454,6 +473,82 @@ usage. Its primary use is for debugging type parameters.
     be POSIX compatible shell.
 
 
+CONFIGURATION FILE
+------------------
+cdist obtains configuration data from the following sources in the following
+order:
+
+    #. command-line options
+    #. configuration file specified at command-line
+    #. configuration file specified in CDIST_CONFIG_FILE environment variable
+    #. environment variables
+    #. user's configuration file (first one found of ~/.cdist.cfg,
+           $XDG_CONFIG_HOME/cdist/cdist.cfg, in specified order)
+    #. system-wide configuration file (/etc/cdist.cfg)
+
+if one exists.
+
+cdist configuration file is in the INI file format. Currently it supports
+only [GLOBAL] section.
+The possible keywords and their meanings are as follows:
+
+:strong:`archiving`
+    Use specified archiving. Valid values include:
+    'none', 'tar', 'tgz', 'tbz2' and 'txz'.
+
+:strong:`beta`
+    Enable beta functionality. It recognizes boolean values from
+    'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'
+
+:strong:`cache_path_pattern`
+    Specify cache path pattern.
+
+:strong:`conf_dir`
+    Comma separated list of configuration directories.
+    If also specified at command line then values from command line are
+    appended to this value.
+
+:strong:`init_manifest`
+    Specify default initial manifest.
+
+:strong:`inventory_dir`
+    Specify inventory directory.
+
+:strong:`jobs`
+    Specify number of jobs for parallel processing. If -1 then the default,
+    number of CPU's in the system is used. If 0 then parallel processing in
+    jobs is disabled. If set to positive number then specified maximum
+    number of processes will be used.
+
+:strong:`local_shell`
+    Shell command used for local execution.
+
+:strong:`out_path`
+    Directory to save cdist output in.
+
+:strong:`parallel`
+    Process hosts in parallel. If -1 then the default, number of CPU's in
+    the system is used. If 0 then parallel processing of hosts is disabled.
+    If set to positive number then specified maximum number of processes
+    will be used.
+
+:strong:`remote_copy`
+    Command to use for remote copy (should behave like scp).
+
+:strong:`remote_exec`
+    Command to use for remote execution (should behave like ssh).
+
+:strong:`remote_out_path`
+    Directory to save cdist output in on the target host.
+
+:strong:`remote_shell`
+    Shell command at remote host used for remote execution.
+
+:strong:`verbosity`
+    Set verbosity level. Valid values are: 
+    'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
+
+
 FILES
 -----
 ~/.cdist
@@ -468,6 +563,10 @@ cdist/conf
 cdist/inventory
     The distribution inventory directory.
     This path is relative to cdist installation directory.
+/etc/cdist.cfg
+    Global cdist configuration file, if exists.
+~/.cdist/cdist.cfg
+    Local cdist configuration file, if exists.
 
 NOTES
 -----
@@ -592,6 +691,10 @@ CDIST_BETA
 CDIST_CACHE_PATH_PATTERN
     Custom cache path pattern.
 
+CDIST_CONFIG_FILE
+    Custom configuration file.
+
+
 EXIT STATUS
 -----------
 The following exit values shall be returned:

scripts/cdist

@@ -22,26 +22,20 @@
 #
 
 import logging
+import cdist
+import cdist.argparse
+import cdist.banner
+import cdist.config
+import cdist.install
+import cdist.shell
+import cdist.inventory
 
 
 def commandline():
     """Parse command line"""
 
-    import cdist.argparse
+    parser, cfg = cdist.argparse.parse_and_configure(sys.argv[1:])
-    import cdist.banner
+    args = cfg.get_args()
-    import cdist.config
-    import cdist.install
-    import cdist.shell
-    import cdist.inventory
-
-    parser = cdist.argparse.get_parsers()
-    args = parser['main'].parse_args(sys.argv[1:])
-
-    # Loglevels are handled globally in here
-    cdist.argparse.handle_loglevel(args)
-
-    log.verbose("version %s" % cdist.VERSION)
-    log.trace(args)
 
     # Work around python 3.3 bug:
     # http://bugs.python.org/issue16308
@@ -58,7 +52,6 @@ def commandline():
         parser['main'].print_help()
         sys.exit(0)
 
-    cdist.argparse.check_beta(vars(args))
     args.func(args)
 
 
@@ -74,9 +67,8 @@ if __name__ == "__main__":
     exit_code = 0
 
     try:
-        import os
         import re
-        import cdist
+        import os
 
         log = logging.getLogger("cdist")