sparrowhawk/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Configfile (#559)

Browse source 
Add cdist configuration/config file support.
This commit is contained in:
Darko Poljak 2017-08-30 23:02:17 +02:00 committed by GitHub
parent 4b2f23db62
commit bdee7273af
11 changed files with 1800 additions and 136 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
cdist/argparse.py
View file

@ -1,9 +1,9 @@
import argparse
import cdist
import multiprocessing
import os
import logging
import collections
import cdist.configuration
# set of beta sub-commands
@ -115,7 +115,7 @@ def get_parsers():
'-b', '--beta',
help=('Enable beta functionality. '),
action='store_true', dest='beta',
default='CDIST_BETA' in os.environ)
default=False)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(
@ -136,12 +136,18 @@ def get_parsers():
'-I', '--inventory',
help=('Use specified custom inventory directory. '
'Inventory directory is set up by the following rules: '
'if this argument is set then specified directory is used, '
'if CDIST_INVENTORY_DIR env var is set then its value is '
'used, if HOME env var is set then ~/.cdist/inventory is '
'if cdist configuration resolves this value then specified '
'directory is used, '
'if HOME env var is set then ~/.cdist/inventory is '
'used, otherwise distribution inventory directory is used.'),
dest="inventory_dir", required=False)
parser['common'] = argparse.ArgumentParser(add_help=False)
parser['common'].add_argument(
'-g', '--config-file',
help=('Use specified custom configuration file.'),
dest="config_file", required=False)
# Config
parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
@ -149,7 +155,7 @@ def get_parsers():
help=('Specify custom cache path pattern. If '
'it is not set then default hostdir is used.'),
dest='cache_path_pattern',
default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
default=None)
parser['config_main'].add_argument(
'-c', '--conf-dir',
help=('Add configuration directory (can be repeated, '
@ -195,13 +201,13 @@ def get_parsers():
'--remote-copy',
help='Command to use for remote copy (should behave like scp).',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
default=None)
parser['config_main'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh).'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
default=None)
# Config
parser['config_args'] = argparse.ArgumentParser(add_help=False)
@ -243,6 +249,7 @@ def get_parsers():
dest='tag', required=False, action="store_true", default=False)
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['config_main'],
parser['inventory_common'],
parser['config_args']])
@ -256,12 +263,14 @@ def get_parsers():
# Inventory
parser['inventory'] = parser['sub'].add_parser(
'inventory', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['inventory_common']])
parser['invsub'] = parser['inventory'].add_subparsers(
title="Inventory commands", dest="subcommand")
parser['add-host'] = parser['invsub'].add_parser(
'add-host', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['inventory_common']])
parser['add-host'].add_argument(
'host', nargs='*', help='Host(s) to add.')
@ -275,6 +284,7 @@ def get_parsers():
parser['add-tag'] = parser['invsub'].add_parser(
'add-tag', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['inventory_common']])
parser['add-tag'].add_argument(
'host', nargs='*',
@ -305,6 +315,7 @@ def get_parsers():
parser['del-host'] = parser['invsub'].add_parser(
'del-host', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['inventory_common']])
parser['del-host'].add_argument(
'host', nargs='*', help='Host(s) to delete.')
@ -321,6 +332,7 @@ def get_parsers():
parser['del-tag'] = parser['invsub'].add_parser(
'del-tag', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['inventory_common']])
parser['del-tag'].add_argument(
'host', nargs='*',
@ -355,6 +367,7 @@ def get_parsers():
parser['list'] = parser['invsub'].add_parser(
'list', parents=[parser['loglevel'], parser['beta'],
parser['common'],
parser['inventory_common']])
parser['list'].add_argument(
'host', nargs='*', help='Host(s) to list.')
@ -401,3 +414,23 @@ def handle_loglevel(args):
args.verbose = _verbosity_level_off
logging.root.setLevel(_verbosity_level[args.verbose])
def parse_and_configure(argv):
parser = get_parsers()
parser_args = parser['main'].parse_args(argv)
cfg = cdist.configuration.Configuration(parser_args)
args = cfg.get_args()
# Loglevels are handled globally in here
handle_loglevel(args)
log = logging.getLogger("cdist")
log.verbose("version %s" % cdist.VERSION)
log.trace('command line args: {}'.format(cfg.command_line_args))
log.trace('configuration: {}'.format(cfg.get_config()))
log.trace('configured args: {}'.format(args))
check_beta(vars(args))
return parser, cfg

26
cdist/config.py
View file

@ -31,15 +31,12 @@ import multiprocessing
from cdist.mputil import mp_pool_run, mp_sig_handler
import atexit
import shutil
import cdist
import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
import cdist.util.ipaddr as ipaddr
import cdist.configuration
from cdist import core, inventory
from cdist.util.remoteutil import inspect_ssh_mux_opts
@ -162,8 +159,11 @@ class Config(object):
hostcnt = 0
cfg = cdist.configuration.Configuration(args)
configuration = cfg.get_config(section='GLOBAL')
if args.tag or args.all_tagged_hosts:
inventory.determine_default_inventory_dir(args)
inventory.determine_default_inventory_dir(args, configuration)
if args.all_tagged_hosts:
inv_list = inventory.InventoryList(
hosts=None, istag=True, hostfile=None,
@ -191,7 +191,7 @@ class Config(object):
else:
# if configuring by host then check inventory for tags
host = entry
inventory.determine_default_inventory_dir(args)
inventory.determine_default_inventory_dir(args, configuration)
inv_list = inventory.InventoryList(
hosts=(host,), db_basedir=args.inventory_dir)
inv = tuple(inv_list.entries())
@ -208,14 +208,16 @@ class Config(object):
hostcnt += 1
if args.parallel:
pargs = (host, host_tags, host_base_path, hostdir, args, True)
pargs = (host, host_tags, host_base_path, hostdir, args, True,
configuration)
log.trace(("Args for multiprocessing operation "
"for host {}: {}".format(host, pargs)))
process_args.append(pargs)
else:
try:
cls.onehost(host, host_tags, host_base_path, hostdir,
args, parallel=False)
args, parallel=False,
configuration=configuration)
except cdist.Error as e:
failed_hosts.append(host)
if args.parallel and len(process_args) == 1:
@ -281,7 +283,7 @@ class Config(object):
@classmethod
def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
parallel):
parallel, configuration):
"""Configure ONE system.
If operating in parallel then return tuple (host, True|False, )
so that main process knows for which host function was successful.
@ -308,7 +310,8 @@ class Config(object):
initial_manifest=args.manifest,
add_conf_dirs=args.conf_dir,
cache_path_pattern=args.cache_path_pattern,
quiet_mode=args.quiet)
quiet_mode=args.quiet,
configuration=configuration)
remote = cdist.exec.remote.Remote(
target_host=target_host,
@ -316,7 +319,8 @@ class Config(object):
remote_copy=remote_copy,
base_path=args.remote_out_path,
quiet_mode=args.quiet,
archiving_mode=args.use_archiving)
archiving_mode=args.use_archiving,
configuration=configuration)
cleanup_cmds = []
if cleanup_cmd:

421
cdist/configuration.py Normal file
View file

@ -0,0 +1,421 @@
# -*- coding: utf-8 -*-
#
# 2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import configparser
import os
import cdist
import cdist.argparse
import re
import multiprocessing
class Singleton(type):
instance = None
def __call__(cls, *args, **kwargs):
if not cls.instance:
cls.instance = super(Singleton, cls).__call__(*args, **kwargs)
return cls.instance
_VERBOSITY_VALUES = (
'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE', 'OFF',
)
_ARCHIVING_VALUES = (
'tar', 'tgz', 'tbz2', 'txz', 'none',
)
class OptionBase:
def __init__(self, name):
self.name = name
def converter(self, *args, **kwargs):
raise NotImplementedError('Subclass should implement this method')
def translate(self, val):
return val
def update_value(self, currval, newval, update_appends=False):
'''Update current option value currval with new option value newval.
If update_appends is True and if currval and newval are lists then
resulting list contains all values in currval plus all values in
newval. Otherwise, newval is returned.
'''
if (isinstance(currval, list) and isinstance(newval, list) and
update_appends):
rv = []
if currval:
rv.extend(currval)
if newval:
rv.extend(newval)
if not rv:
rv = None
return rv
else:
return newval
class StringOption(OptionBase):
def __init__(self, name):
super().__init__(name)
def converter(self):
def string_converter(val):
return self.translate(str(val))
return string_converter
def translate(self, val):
if val:
return val
else:
return None
class BooleanOption(OptionBase):
BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES
def __init__(self, name):
super().__init__(name)
def converter(self):
def boolean_converter(val):
v = val.lower()
if v not in self.BOOLEAN_STATES:
raise ValueError('Invalid {} boolean value: {}'.format(
self.name, val))
return self.translate(v)
return boolean_converter
def translate(self, val):
return self.BOOLEAN_STATES[val]
class IntOption(OptionBase):
def __init__(self, name):
super().__init__(name)
def converter(self):
def int_converter(val):
return self.translate(int(val))
return int_converter
class LowerBoundIntOption(IntOption):
def __init__(self, name, lower_bound):
super().__init__(name)
self.lower_bound = lower_bound
def converter(self):
def lower_bound_converter(val):
converted = super(LowerBoundIntOption, self).converter()(val)
if converted < self.lower_bound:
raise ValueError("Invalid {} value: {} < {}".format(
self.name, val, self.lower_bound))
return converted
return lower_bound_converter
class SpecialCasesLowerBoundIntOption(LowerBoundIntOption):
def __init__(self, name, lower_bound, special_cases_mapping):
super().__init__(name, lower_bound)
self.special_cases_mapping = special_cases_mapping
def translate(self, val):
if val in self.special_cases_mapping:
return self.special_cases_mapping[val]
else:
return val
class JobsOption(SpecialCasesLowerBoundIntOption):
def __init__(self, name):
super().__init__(name, -1, {-1: multiprocessing.cpu_count()})
class SelectOption(OptionBase):
def __init__(self, name, valid_values):
super().__init__(name)
self.valid_values = valid_values
def converter(self):
def select_converter(val):
if val in self.valid_values:
return self.translate(val)
else:
raise ValueError("Invalid {} value: {}.".format(
self.name, val))
return select_converter
class VerbosityOption(SelectOption):
def __init__(self):
super().__init__('verbosity', _VERBOSITY_VALUES)
def translate(self, val):
name = 'VERBOSE_' + val
verbose = getattr(cdist.argparse, name)
return verbose
class DelimitedValuesOption(OptionBase):
def __init__(self, name, delimiter):
super().__init__(name)
self.delimiter = delimiter
def converter(self):
def delimited_values_converter(val):
vals = re.split(r'(?<!\\)' + self.delimiter, val)
vals = [x for x in vals if x]
return self.translate(vals)
return delimited_values_converter
def translate(self, val):
if val:
return val
else:
return None
class ConfDirOption(DelimitedValuesOption):
def __init__(self):
super().__init__('conf_dir', os.pathsep)
class ArchivingOption(SelectOption):
def __init__(self):
super().__init__('archiving', _ARCHIVING_VALUES)
def translate(self, val):
if val == 'none':
return None
else:
return val
_ARG_OPTION_MAPPING = {
'beta': 'beta',
'cache_path_pattern': 'cache_path_pattern',
'conf_dir': 'conf_dir',
'manifest': 'init_manifest',
'out_path': 'out_path',
'remote_out_path': 'remote_out_path',
'remote_copy': 'remote_copy',
'remote_exec': 'remote_exec',
'inventory_dir': 'inventory_dir',
'jobs': 'jobs',
'parallel': 'parallel',
'verbose': 'verbosity',
'use_archiving': 'archiving',
}
class Configuration(metaclass=Singleton):
_config_basename = 'cdist.cfg'
_global_config_file = os.path.join('/', 'etc', _config_basename, )
_local_config_file = os.path.join(os.path.expanduser('~'),
'.' + _config_basename, )
if (not (os.path.exists(_local_config_file) and
os.path.isfile(_local_config_file))):
_local_config_file = os.path.join(
os.environ.get('XDG_CONFIG_HOME',
os.path.expanduser('~/.config/cdist')),
_config_basename)
default_config_files = (_global_config_file, _local_config_file, )
ENV_VAR_CONFIG_FILE = 'CDIST_CONFIG_FILE'
VERBOSITY_VALUES = _VERBOSITY_VALUES
ARCHIVING_VALUES = _ARCHIVING_VALUES
CONFIG_FILE_OPTIONS = {
'GLOBAL': {
'beta': BooleanOption('beta'),
'local_shell': StringOption('local_shell'),
'remote_shell': StringOption('remote_shell'),
'cache_path_pattern': StringOption('cache_path_pattern'),
'conf_dir': ConfDirOption(),
'init_manifest': StringOption('init_manifest'),
'out_path': StringOption('out_path'),
'remote_out_path': StringOption('remote_out_path'),
'remote_copy': StringOption('remote_copy'),
'remote_exec': StringOption('remote_exec'),
'inventory_dir': StringOption('inventory_dir'),
'jobs': JobsOption('jobs'),
'parallel': JobsOption('parallel'),
'verbosity': VerbosityOption(),
'archiving': ArchivingOption(),
},
}
ENV_VAR_OPTION_MAPPING = {
'CDIST_BETA': 'beta',
'CDIST_PATH': 'conf_dir',
'CDIST_LOCAL_SHELL': 'local_shell',
'CDIST_REMOTE_SHELL': 'remote_shell',
'CDIST_REMOTE_EXEC': 'remote_exec',
'CDIST_REMOTE_COPY': 'remote_copy',
'CDIST_INVENTORY_DIR': 'inventory_dir',
'CDIST_CACHE_PATH_PATTERN': 'cache_path_pattern',
}
ENV_VAR_BOOLEAN_OPTIONS = ('CDIST_BETA', )
ARG_OPTION_MAPPING = _ARG_OPTION_MAPPING
ADJUST_ARG_OPTION_MAPPING = {
_ARG_OPTION_MAPPING[key]: key for key in _ARG_OPTION_MAPPING
}
def _convert_args(self, args):
if args:
if hasattr(args, '__dict__'):
return vars(args)
else:
raise ValueError(
'args parameter must be have __dict__ attribute')
else:
return None
def __init__(self, command_line_args, env=os.environ,
config_files=default_config_files):
self.command_line_args = command_line_args
self.args = self._convert_args(command_line_args)
self.env = env
self.config_files = config_files
self.config = self._get_config()
def get_config(self, section=None):
if section is None:
return self.config
if section in self.config:
return self.config[section]
raise ValueError('Unknown section: {}'.format(section))
def _get_args_name_value(self, arg_name, val):
if arg_name == 'verbosity' and val == 'OFF':
name = 'quiet'
rv = True
else:
name = arg_name
rv = val
return (name, rv)
def get_args(self, section='GLOBAL'):
args = self.command_line_args
cfg = self.get_config(section)
for option in self.ADJUST_ARG_OPTION_MAPPING:
if option in cfg:
arg_name = self.ADJUST_ARG_OPTION_MAPPING[option]
val = cfg[option]
name, val = self._get_args_name_value(arg_name, val)
setattr(args, name, val)
return args
def _read_config_file(self, files):
config_parser = configparser.ConfigParser()
config_parser.read(files)
d = dict()
for section in config_parser.sections():
if section not in self.CONFIG_FILE_OPTIONS:
raise ValueError("Invalid section: {}.".format(section))
if section not in d:
d[section] = dict()
for option in config_parser[section]:
if option not in self.CONFIG_FILE_OPTIONS[section]:
raise ValueError("Invalid option: {}.".format(option))
option_object = self.CONFIG_FILE_OPTIONS[section][option]
converter = option_object.converter()
val = config_parser[section][option]
newval = converter(val)
d[section][option] = newval
return d
def _read_env_var_config(self, env, section):
d = dict()
for option in self.ENV_VAR_OPTION_MAPPING:
if option in env:
dst_opt = self.ENV_VAR_OPTION_MAPPING[option]
if option in self.ENV_VAR_BOOLEAN_OPTIONS:
d[dst_opt] = True
else:
option_object = self.CONFIG_FILE_OPTIONS[section][dst_opt]
converter = option_object.converter()
val = env[option]
newval = converter(val)
d[dst_opt] = newval
return d
def _read_args_config(self, args):
d = dict()
for option in self.ARG_OPTION_MAPPING:
if option in args:
dst_opt = self.ARG_OPTION_MAPPING[option]
d[dst_opt] = args[option]
return d
def _update_config_dict(self, config, newconfig, update_appends=False):
for section in newconfig:
self._update_config_dict_section(
section, config, newconfig[section], update_appends)
def _update_config_dict_section(self, section, config, newconfig,
update_appends=False):
if section not in config:
config[section] = dict()
for option in newconfig:
newval = newconfig[option]
if option in config[section]:
currval = config[section][option]
else:
currval = None
option_object = self.CONFIG_FILE_OPTIONS[section][option]
config[section][option] = option_object.update_value(
currval, newval, update_appends)
def _get_config(self):
# global config file
# local config file
config = self._read_config_file(self.config_files)
# default empty config if needed
if not config:
config['GLOBAL'] = dict()
# environment variables
newconfig = self._read_env_var_config(self.env, 'GLOBAL')
for section in config:
self._update_config_dict_section(section, config, newconfig)
# config file in CDIST_CONFIG_FILE env var
config_file = os.environ.get(self.ENV_VAR_CONFIG_FILE, None)
if config_file:
newconfig = self._read_config_file(config_file)
self._update_config_dict(config, newconfig)
# command line config file
if (self.args and 'config_file' in self.args and
self.args['config_file']):
newconfig = self._read_config_file(self.args['config_file'])
self._update_config_dict(config, newconfig)
# command line
if self.args:
newconfig = self._read_args_config(self.args)
for section in config:
self._update_config_dict_section(section, config, newconfig,
update_appends=True)
return config

28
cdist/exec/local.py
View file

@ -55,7 +55,8 @@ class Local(object):
initial_manifest=None,
add_conf_dirs=None,
cache_path_pattern=None,
quiet_mode=False):
quiet_mode=False,
configuration=None):
self.target_host = target_host
if target_host_tags is None:
@ -70,6 +71,10 @@ class Local(object):
self._add_conf_dirs = add_conf_dirs
self.cache_path_pattern = cache_path_pattern
self.quiet_mode = quiet_mode
if configuration:
self.configuration = configuration
else:
self.configuration = {}
self._init_log()
self._init_permissions()
@ -142,10 +147,12 @@ class Local(object):
self.conf_dirs.append(self.home_dir)
# Add directories defined in the CDIST_PATH environment variable
if 'CDIST_PATH' in os.environ:
cdist_path_dirs = re.split(r'(?<!\\):', os.environ['CDIST_PATH'])
cdist_path_dirs.reverse()
self.conf_dirs.extend(cdist_path_dirs)
# if 'CDIST_PATH' in os.environ:
# cdist_path_dirs = re.split(r'(?<!\\):', os.environ['CDIST_PATH'])
# cdist_path_dirs.reverse()
# self.conf_dirs.extend(cdist_path_dirs)
if 'conf_dir' in self.configuration:
self.conf_dirs.extend(self.configuration['conf_dir'])
# Add command line supplied directories
if self._add_conf_dirs:
@ -175,12 +182,11 @@ class Local(object):
def _init_cache_dir(self, cache_dir):
if cache_dir:
self.cache_path = cache_dir
elif self.home_dir:
self.cache_path = os.path.join(self.home_dir, "cache")
else:
if self.home_dir:
self.cache_path = os.path.join(self.home_dir, "cache")
else:
raise cdist.Error(
"No homedir setup and no cache dir location given")
raise cdist.Error(
"No homedir setup and no cache dir location given")
def rmdir(self, path):
"""Remove directory on the local side."""
@ -258,7 +264,7 @@ class Local(object):
self.log.debug('%s is executable, running it', script)
command = [script]
else:
command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
command = [self.configuration.get('local_shell', "/bin/sh"), "-e"]
self.log.debug('%s is NOT executable, running it with %s',
script, " ".join(command))
command.append(script)

12
cdist/exec/remote.py
View file

@ -62,7 +62,8 @@ class Remote(object):
remote_copy,
base_path=None,
quiet_mode=None,
archiving_mode=None):
archiving_mode=None,
configuration=None):
self.target_host = target_host
self._exec = remote_exec
self._copy = remote_copy
@ -73,6 +74,10 @@ class Remote(object):
self.base_path = "/var/lib/cdist"
self.quiet_mode = quiet_mode
self.archiving_mode = archiving_mode
if configuration:
self.configuration = configuration
else:
self.configuration = {}
self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object")
@ -235,7 +240,10 @@ class Remote(object):
"""
command = [os.environ.get('CDIST_REMOTE_SHELL', "/bin/sh"), "-e"]
command = [
self.configuration.get('CDIST_REMOTE_SHELL', "/bin/sh"),
"-e"
]
command.append(script)
return self.run(command, env, return_output)

67
cdist/inventory.py
View file

@ -26,6 +26,7 @@ import os
import os.path
import itertools
import sys
import cdist.configuration
from cdist.hostsource import hostfile_process_line
DIST_INVENTORY_DB_NAME = "inventory"
@ -34,21 +35,23 @@ dist_inventory_db = os.path.abspath(os.path.join(
os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME))
def determine_default_inventory_dir(args):
def determine_default_inventory_dir(args, configuration):
# The order of inventory dir setting by decreasing priority
# 1. inventory_dir argument
# 2. CDIST_INVENTORY_DIR env var if set
# 3. ~/.cdist/inventory if HOME env var is set
# 4. distribution inventory directory
if not args.inventory_dir:
if 'CDIST_INVENTORY_DIR' in os.environ:
args.inventory_dir = os.environ['CDIST_INVENTORY_DIR']
# 1. inventory_dir from configuration
# 2. ~/.cdist/inventory if HOME env var is set
# 3. distribution inventory directory
inventory_dir_set = False
if 'inventory_dir' in configuration:
val = configuration['inventory_dir']
if val:
args.inventory_dir = val
inventory_dir_set = True
if not inventory_dir_set:
home = cdist.home_dir()
if home:
args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
else:
home = cdist.home_dir()
if home:
args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
else:
args.inventory_dir = dist_inventory_db
args.inventory_dir = dist_inventory_db
def contains_all(big, little):
@ -80,8 +83,12 @@ def rstrip_nl(s):
class Inventory(object):
"""Inventory main class"""
def __init__(self, db_basedir=dist_inventory_db):
def __init__(self, db_basedir=dist_inventory_db, configuration=None):
self.db_basedir = db_basedir
if configuration:
self.configuration = configuration
else:
self.configuration = {}
self.log = logging.getLogger("inventory")
self.init_db()
@ -171,7 +178,10 @@ class Inventory(object):
log = logging.getLogger("inventory")
if 'taglist' in args:
args.taglist = cls.strlist_to_list(args.taglist)
determine_default_inventory_dir(args)
cfg = cdist.configuration.Configuration(args)
configuration = cfg.get_config(section='GLOBAL')
determine_default_inventory_dir(args, configuration)
log.debug("Using inventory: {}".format(args.inventory_dir))
log.trace("Inventory args: {}".format(vars(args)))
@ -182,23 +192,26 @@ class Inventory(object):
hostfile=args.hostfile,
db_basedir=args.inventory_dir,
list_only_host=args.list_only_host,
has_all_tags=args.has_all_tags)
has_all_tags=args.has_all_tags,
configuration=configuration)
elif args.subcommand == "add-host":
c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
db_basedir=args.inventory_dir)
db_basedir=args.inventory_dir,
configuration=configuration)
elif args.subcommand == "del-host":
c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
all=args.all, db_basedir=args.inventory_dir,
action="del")
action="del", configuration=configuration)
elif args.subcommand == "add-tag":
c = InventoryTag(hosts=args.host, tags=args.taglist,
hostfile=args.hostfile, tagfile=args.tagfile,
db_basedir=args.inventory_dir)
db_basedir=args.inventory_dir,
configuration=configuration)
elif args.subcommand == "del-tag":
c = InventoryTag(hosts=args.host, tags=args.taglist,
hostfile=args.hostfile, tagfile=args.tagfile,
all=args.all, db_basedir=args.inventory_dir,
action="del")
action="del", configuration=configuration)
else:
raise cdist.Error("Unknown inventory command \'{}\'".format(
args.subcommand))
@ -208,8 +221,8 @@ class Inventory(object):
class InventoryList(Inventory):
def __init__(self, hosts=None, istag=False, hostfile=None,
list_only_host=False, has_all_tags=False,
db_basedir=dist_inventory_db):
super().__init__(db_basedir)
db_basedir=dist_inventory_db, configuration=None):
super().__init__(db_basedir, configuration)
self.hosts = hosts
self.istag = istag
self.hostfile = hostfile
@ -274,8 +287,9 @@ class InventoryList(Inventory):
class InventoryHost(Inventory):
def __init__(self, hosts=None, hostfile=None,
db_basedir=dist_inventory_db, all=False, action="add"):
super().__init__(db_basedir)
db_basedir=dist_inventory_db, all=False, action="add",
configuration=None):
super().__init__(db_basedir, configuration)
self.actions = ("add", "del")
if action not in self.actions:
raise cdist.Error("Invalid action \'{}\', valid actions are:"
@ -323,8 +337,9 @@ class InventoryHost(Inventory):
class InventoryTag(Inventory):
def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None,
db_basedir=dist_inventory_db, all=False, action="add"):
super().__init__(db_basedir)
db_basedir=dist_inventory_db, all=False, action="add",
configuration=None):
super().__init__(db_basedir, configuration)
self.actions = ("add", "del")
if action not in self.actions:
raise cdist.Error("Invalid action \'{}\', valid actions are:"

1076
cdist/test/configuration/__init__.py Normal file
View file

File diff suppressed because it is too large Load diff

0
cdist/test/configuration/fixtures/.nonempty Normal file
View file

6
cdist/test/exec/local.py
View file

@ -28,8 +28,10 @@ import string
import random
import time
import datetime
import argparse
import cdist
import cdist.configuration as cc
from cdist import test
from cdist.exec import local
@ -165,6 +167,9 @@ class LocalTestCase(test.CdistTestCase):
os.environ['CDIST_PATH'] = conf_dir
configuration = cc.Configuration(argparse.Namespace(),
env=os.environ)
link_test_local = local.Local(
target_host=(
'localhost',
@ -174,6 +179,7 @@ class LocalTestCase(test.CdistTestCase):
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=test.cdist_exec_path,
configuration=configuration
)
link_test_local._create_conf_path_and_link_conf_dirs()

221
docs/src/man1/cdist.rst
View file

@ -15,43 +15,45 @@ SYNOPSIS
cdist banner [-h] [-q] [-v]
cdist config [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
[-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]]
[-s] [-t]
cdist config [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
[-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
[--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
[-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]]
cdist install [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
[-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_DIR]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] [-p [HOST_MAX]]
[-s] [-t]
cdist install [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST]
[-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
[--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
[-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[host [host ...]]
cdist inventory [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
cdist inventory [-h] [-q] [-v] [-b] [-g CONFIG_FILE] [-I INVENTORY_DIR]
{add-host,add-tag,del-host,del-tag,list} ...
cdist inventory add-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
[-f HOSTFILE]
cdist inventory add-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-f HOSTFILE]
[host [host ...]]
cdist inventory add-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
[-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
cdist inventory add-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-f HOSTFILE] [-T TAGFILE]
[-t TAGLIST]
[host [host ...]]
cdist inventory del-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE]
cdist inventory del-host [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE]
[host [host ...]]
cdist inventory del-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
cdist inventory del-tag [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE]
[-T TAGFILE] [-t TAGLIST]
[host [host ...]]
cdist inventory list [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
[-f HOSTFILE] [-H] [-t]
cdist inventory list [-h] [-q] [-v] [-b] [-g CONFIG_FILE]
[-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t]
[host [host ...]]
cdist shell [-h] [-q] [-v] [-s SHELL]
@ -133,15 +135,18 @@ Install command is currently in beta.
read hosts from stdin. For the file format see
:strong:`HOSTFILE FORMAT` below.
.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
Use specified custom configuration file.
.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
Use specified custom inventory directory. Inventory
directory is set up by the following rules: if this
argument is set then specified directory is used, if
CDIST_INVENTORY_DIR env var is set then its value is
used, if HOME env var is set then ~/.cdit/inventory is
used, otherwise distribution inventory directory is
used.
directory is set up by the following rules: if cdist
configuration resolves this value then specified
directory is used, if HOME env var is set then
~/.cdit/inventory is used, otherwise distribution
inventory directory is used.
.. option:: -i MANIFEST, --initial-manifest MANIFEST
@ -254,16 +259,18 @@ Add host(s) to inventory database.
host or host file is specified then, by default, read
from stdin. Hostfile format is the same as config hostfile format.
.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
Use specified custom configuration file.
.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
Use specified custom inventory directory. Inventory
directory is set up by the following rules: if this
argument is set then specified directory is used, if
CDIST_INVENTORY_DIR env var is set then its value is
used, if HOME env var is set then ~/.cdist/inventory is
used, otherwise distribution inventory directory is
used.
directory is set up by the following rules: if cdist
configuration resolves this value then specified
directory is used, if HOME env var is set then
~/.cdit/inventory is used, otherwise distribution
inventory directory is used.
INVENTORY ADD-TAG
@ -287,15 +294,18 @@ Add tag(s) to inventory database.
are specified then tags are read from stdin and are
added to all hosts. Hostfile format is the same as config hostfile format.
.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
Use specified custom configuration file.
.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
Use specified custom inventory directory. Inventory
directory is set up by the following rules: if this
argument is set then specified directory is used, if
CDIST_INVENTORY_DIR env var is set then its value is
used, if HOME env var is set then ~/.cdist/inventory is
used, otherwise distribution inventory directory is
used.
directory is set up by the following rules: if cdist
configuration resolves this value then specified
directory is used, if HOME env var is set then
~/.cdit/inventory is used, otherwise distribution
inventory directory is used.
.. option:: -T TAGFILE, --tag-file TAGFILE
@ -335,15 +345,18 @@ Delete host(s) from inventory database.
host or host file is specified then, by default, read
from stdin. Hostfile format is the same as config hostfile format.
.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
Use specified custom configuration file.
.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
Use specified custom inventory directory. Inventory
directory is set up by the following rules: if this
argument is set then specified directory is used, if
CDIST_INVENTORY_DIR env var is set then its value is
used, if HOME env var is set then ~/.cdist/inventory is
used, otherwise distribution inventory directory is
used.
directory is set up by the following rules: if cdist
configuration resolves this value then specified
directory is used, if HOME env var is set then
~/.cdit/inventory is used, otherwise distribution
inventory directory is used.
INVENTORY DEL-TAG
@ -372,15 +385,18 @@ Delete tag(s) from inventory database.
from stdin and are deleted from all hosts. Hostfile
format is the same as config hostfile format.
.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
Use specified custom configuration file.
.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
Use specified custom inventory directory. Inventory
directory is set up by the following rules: if this
argument is set then specified directory is used, if
CDIST_INVENTORY_DIR env var is set then its value is
used, if HOME env var is set then ~/.cdist/inventory is
used, otherwise distribution inventory directory is
used.
directory is set up by the following rules: if cdist
configuration resolves this value then specified
directory is used, if HOME env var is set then
~/.cdit/inventory is used, otherwise distribution
inventory directory is used.
.. option:: -T TAGFILE, --tag-file TAGFILE
@ -421,6 +437,10 @@ List inventory database.
host or host file is specified then, by default, list
all. Hostfile format is the same as config hostfile format.
.. option:: -g CONFIG_FILE, --config-file CONFIG_FILE
Use specified custom configuration file.
.. option:: -H, --host-only
Suppress tags listing.
@ -428,12 +448,11 @@ List inventory database.
.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
Use specified custom inventory directory. Inventory
directory is set up by the following rules: if this
argument is set then specified directory is used, if
CDIST_INVENTORY_DIR env var is set then its value is
used, if HOME env var is set then ~/.cdist/inventory is
used, otherwise distribution inventory directory is
used.
directory is set up by the following rules: if cdist
configuration resolves this value then specified
directory is used, if HOME env var is set then
~/.cdit/inventory is used, otherwise distribution
inventory directory is used.
.. option:: -t, --tag
@ -454,6 +473,82 @@ usage. Its primary use is for debugging type parameters.
be POSIX compatible shell.
CONFIGURATION FILE
------------------
cdist obtains configuration data from the following sources in the following
order:
#. command-line options
#. configuration file specified at command-line
#. configuration file specified in CDIST_CONFIG_FILE environment variable
#. environment variables
#. user's configuration file (first one found of ~/.cdist.cfg,
$XDG_CONFIG_HOME/cdist/cdist.cfg, in specified order)
#. system-wide configuration file (/etc/cdist.cfg)
if one exists.
cdist configuration file is in the INI file format. Currently it supports
only [GLOBAL] section.
The possible keywords and their meanings are as follows:
:strong:`archiving`
Use specified archiving. Valid values include:
'none', 'tar', 'tgz', 'tbz2' and 'txz'.
:strong:`beta`
Enable beta functionality. It recognizes boolean values from
'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'
:strong:`cache_path_pattern`
Specify cache path pattern.
:strong:`conf_dir`
Comma separated list of configuration directories.
If also specified at command line then values from command line are
appended to this value.
:strong:`init_manifest`
Specify default initial manifest.
:strong:`inventory_dir`
Specify inventory directory.
:strong:`jobs`
Specify number of jobs for parallel processing. If -1 then the default,
number of CPU's in the system is used. If 0 then parallel processing in
jobs is disabled. If set to positive number then specified maximum
number of processes will be used.
:strong:`local_shell`
Shell command used for local execution.
:strong:`out_path`
Directory to save cdist output in.
:strong:`parallel`
Process hosts in parallel. If -1 then the default, number of CPU's in
the system is used. If 0 then parallel processing of hosts is disabled.
If set to positive number then specified maximum number of processes
will be used.
:strong:`remote_copy`
Command to use for remote copy (should behave like scp).
:strong:`remote_exec`
Command to use for remote execution (should behave like ssh).
:strong:`remote_out_path`
Directory to save cdist output in on the target host.
:strong:`remote_shell`
Shell command at remote host used for remote execution.
:strong:`verbosity`
Set verbosity level. Valid values are:
'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
FILES
-----
~/.cdist
@ -468,6 +563,10 @@ cdist/conf
cdist/inventory
The distribution inventory directory.
This path is relative to cdist installation directory.
/etc/cdist.cfg
Global cdist configuration file, if exists.
~/.cdist/cdist.cfg
Local cdist configuration file, if exists.
NOTES
-----
@ -592,6 +691,10 @@ CDIST_BETA
CDIST_CACHE_PATH_PATTERN
Custom cache path pattern.
CDIST_CONFIG_FILE
Custom configuration file.
EXIT STATUS
-----------
The following exit values shall be returned:

28
scripts/cdist
View file

@ -22,26 +22,20 @@
#
import logging
import cdist
import cdist.argparse
import cdist.banner
import cdist.config
import cdist.install
import cdist.shell
import cdist.inventory
def commandline():
"""Parse command line"""
import cdist.argparse
import cdist.banner
import cdist.config
import cdist.install
import cdist.shell
import cdist.inventory
parser = cdist.argparse.get_parsers()
args = parser['main'].parse_args(sys.argv[1:])
# Loglevels are handled globally in here
cdist.argparse.handle_loglevel(args)
log.verbose("version %s" % cdist.VERSION)
log.trace(args)
parser, cfg = cdist.argparse.parse_and_configure(sys.argv[1:])
args = cfg.get_args()
# Work around python 3.3 bug:
# http://bugs.python.org/issue16308
@ -58,7 +52,6 @@ def commandline():
parser['main'].print_help()
sys.exit(0)
cdist.argparse.check_beta(vars(args))
args.func(args)
@ -74,9 +67,8 @@ if __name__ == "__main__":
exit_code = 0
try:
import os
import re
import cdist
import os
log = logging.getLogger("cdist")