diff --git a/.gitignore b/.gitignore index 4258c2eb..55374d6d 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,9 @@ docs/src/cdist-reference.rst # Ignore cdist cache for version control /cache/ +# Ignore inventory basedir +cdist/inventory/ + # Python: cache, distutils, distribution in general __pycache__/ *.pyc diff --git a/cdist/argparse.py b/cdist/argparse.py index 8208d1f0..16b9d054 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -7,10 +7,10 @@ import collections # set of beta sub-commands -BETA_COMMANDS = set(('install', )) +BETA_COMMANDS = set(('install', 'inventory', )) # set of beta arguments for sub-commands BETA_ARGS = { - 'config': set(('jobs', )), + 'config': set(('jobs', 'tag', 'all_tagged_hosts', )), } EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" # Parser others can reuse @@ -121,6 +121,17 @@ def get_parsers(): 'banner', parents=[parser['loglevel']]) parser['banner'].set_defaults(func=cdist.banner.banner) + parser['inventory_common'] = argparse.ArgumentParser(add_help=False) + parser['inventory_common'].add_argument( + '-I', '--inventory', + help=('Use specified custom inventory directory. ' + 'Inventory directory is set up by the following rules: ' + 'if this argument is set then specified directory is used, ' + 'if CDIST_INVENTORY_DIR env var is set then its value is ' + 'used, if HOME env var is set then ~/.cdist/inventory is ' + 'used, otherwise distribution inventory directory is used.'), + dest="inventory_dir", required=False) + # Config parser['config_main'] = argparse.ArgumentParser(add_help=False) parser['config_main'].add_argument( @@ -156,6 +167,10 @@ def get_parsers(): # remote-copy and remote-exec defaults are environment variables # if set; if not then None - these will be futher handled after # parsing to determine implementation default + parser['config_main'].add_argument( + '-r', '--remote-out-dir', + help='Directory to save cdist output in on the target host', + dest="remote_out_path") parser['config_main'].add_argument( '--remote-copy', help='Command to use for remote copy (should behave like scp)', @@ -170,6 +185,15 @@ def get_parsers(): # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) + parser['config_args'].add_argument( + '-A', '--all-tagged', + help=('use all hosts present in tags db'), + action="store_true", dest="all_tagged_hosts", default=False) + parser['config_args'].add_argument( + '-a', '--all', + help=('list hosts that have all specified tags, ' + 'if -t/--tag is specified'), + action="store_true", dest="has_all_tags", default=False) parser['config_args'].add_argument( 'host', nargs='*', help='host(s) to operate on') parser['config_args'].add_argument( @@ -183,17 +207,19 @@ def get_parsers(): '-p', '--parallel', help='operate on multiple hosts in parallel', action='store_true', dest='parallel') - parser['config_args'].add_argument( - '-r', '--remote-out-dir', - help='Directory to save cdist output in on the target host', - dest="remote_out_path") parser['config_args'].add_argument( '-s', '--sequential', help='operate on multiple hosts sequentially (default)', action='store_false', dest='parallel') + parser['config_args'].add_argument( + '-t', '--tag', + help=('host is specified by tag, not hostname/address; ' + 'list all hosts that contain any of specified tags'), + dest='tag', required=False, action="store_true", default=False) parser['config'] = parser['sub'].add_parser( 'config', parents=[parser['loglevel'], parser['beta'], parser['config_main'], + parser['inventory_common'], parser['config_args']]) parser['config'].set_defaults(func=cdist.config.Config.commandline) @@ -202,6 +228,134 @@ def get_parsers(): parents=[parser['config']]) parser['install'].set_defaults(func=cdist.install.Install.commandline) + # Inventory + parser['inventory'] = parser['sub'].add_parser( + 'inventory', parents=[parser['loglevel'], parser['beta'], + parser['inventory_common']]) + parser['invsub'] = parser['inventory'].add_subparsers( + title="Inventory commands", dest="subcommand") + + parser['add-host'] = parser['invsub'].add_parser( + 'add-host', parents=[parser['loglevel'], parser['beta'], + parser['inventory_common']]) + parser['add-host'].add_argument( + 'host', nargs='*', help='host(s) to add') + parser['add-host'].add_argument( + '-f', '--file', + help=('Read additional hosts to add from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin.'), + dest='hostfile', required=False) + + parser['add-tag'] = parser['invsub'].add_parser( + 'add-tag', parents=[parser['loglevel'], parser['beta'], + parser['inventory_common']]) + parser['add-tag'].add_argument( + 'host', nargs='*', + help='list of host(s) for which tags are added') + parser['add-tag'].add_argument( + '-f', '--file', + help=('Read additional hosts to add tags from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor hosts/hostfile' + ' are specified then tags are read from stdin and are' + ' added to all hosts.'), + dest='hostfile', required=False) + parser['add-tag'].add_argument( + '-T', '--tag-file', + help=('Read additional tags to add from specified file ' + 'or from stdin if \'-\' (each tag on separate line). ' + 'If no tag or tag file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor hosts/hostfile' + ' are specified then tags are read from stdin and are' + ' added to all hosts.'), + dest='tagfile', required=False) + parser['add-tag'].add_argument( + '-t', '--taglist', + help=("Tag list to be added for specified host(s), comma separated" + " values"), + dest="taglist", required=False) + + parser['del-host'] = parser['invsub'].add_parser( + 'del-host', parents=[parser['loglevel'], parser['beta'], + parser['inventory_common']]) + parser['del-host'].add_argument( + 'host', nargs='*', help='host(s) to delete') + parser['del-host'].add_argument( + '-a', '--all', help=('Delete all hosts'), + dest='all', required=False, action="store_true", default=False) + parser['del-host'].add_argument( + '-f', '--file', + help=('Read additional hosts to delete from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin.'), + dest='hostfile', required=False) + + parser['del-tag'] = parser['invsub'].add_parser( + 'del-tag', parents=[parser['loglevel'], parser['beta'], + parser['inventory_common']]) + parser['del-tag'].add_argument( + 'host', nargs='*', + help='list of host(s) for which tags are deleted') + parser['del-tag'].add_argument( + '-a', '--all', + help=('Delete all tags for specified host(s)'), + dest='all', required=False, action="store_true", default=False) + parser['del-tag'].add_argument( + '-f', '--file', + help=('Read additional hosts to delete tags for from specified ' + 'file or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor hosts/hostfile' + ' are specified then tags are read from stdin and are' + ' deleted from all hosts.'), + dest='hostfile', required=False) + parser['del-tag'].add_argument( + '-T', '--tag-file', + help=('Read additional tags from specified file ' + 'or from stdin if \'-\' (each tag on separate line). ' + 'If no tag or tag file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor' + ' hosts/hostfile are specified then tags are read from' + ' stdin and are added to all hosts.'), + dest='tagfile', required=False) + parser['del-tag'].add_argument( + '-t', '--taglist', + help=("Tag list to be deleted for specified host(s), " + "comma separated values"), + dest="taglist", required=False) + + parser['list'] = parser['invsub'].add_parser( + 'list', parents=[parser['loglevel'], parser['beta'], + parser['inventory_common']]) + parser['list'].add_argument( + 'host', nargs='*', help='host(s) to list') + parser['list'].add_argument( + '-a', '--all', + help=('list hosts that have all specified tags, ' + 'if -t/--tag is specified'), + action="store_true", dest="has_all_tags", default=False) + parser['list'].add_argument( + '-f', '--file', + help=('Read additional hosts to list from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'list all.'), dest='hostfile', required=False) + parser['list'].add_argument( + '-H', '--host-only', help=('Suppress tags listing'), + action="store_true", dest="list_only_host", default=False) + parser['list'].add_argument( + '-t', '--tag', + help=('host is specified by tag, not hostname/address; ' + 'list all hosts that contain any of specified tags'), + action="store_true", default=False) + + parser['inventory'].set_defaults( + func=cdist.inventory.Inventory.commandline) + # Shell parser['shell'] = parser['sub'].add_parser( 'shell', parents=[parser['loglevel']]) diff --git a/cdist/conf/type/__install_stage/man.rst b/cdist/conf/type/__install_stage/man.rst index 6c68c543..e33e1e90 100644 --- a/cdist/conf/type/__install_stage/man.rst +++ b/cdist/conf/type/__install_stage/man.rst @@ -17,9 +17,9 @@ REQUIRED PARAMETERS uri The uri from which to fetch the tarball. Can be anything understood by curl, e.g: - | http://path/to/stage.tgz - | tftp:///path/to/stage.tgz - | file:///local/path/stage.tgz + | http://path/to/stage.tgz + | tftp:///path/to/stage.tgz + | file:///local/path/stage.tgz OPTIONAL PARAMETERS diff --git a/cdist/conf/type/__timezone/explorer/timezone_is b/cdist/conf/type/__timezone/explorer/timezone_is new file mode 100755 index 00000000..4e918121 --- /dev/null +++ b/cdist/conf/type/__timezone/explorer/timezone_is @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2017 Ander Punnar (cdist at kvlt.ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +[ -f /etc/timezone ] && cat /etc/timezone diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote index d72da918..e512f861 100755 --- a/cdist/conf/type/__timezone/gencode-remote +++ b/cdist/conf/type/__timezone/gencode-remote @@ -20,11 +20,16 @@ # # This type allows to configure the desired localtime timezone. -timezone="$__object_id" +timezone_is=$(cat "$__object/explorer/timezone_is") +timezone_should="$__object_id" os=$(cat "$__global/explorer/os") +if [ "$timezone_is" = "$timezone_should" ]; then + exit 0 +fi + case "$os" in ubuntu|debian|devuan) - echo "echo \"$timezone\" > /etc/timezone" + echo "echo \"$timezone_should\" > /etc/timezone" ;; esac diff --git a/cdist/config.py b/cdist/config.py index 48284e8e..cd2f44ae 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -38,6 +38,9 @@ import cdist.hostsource import cdist.exec.local import cdist.exec.remote + +from cdist import inventory + import cdist.util.ipaddr as ipaddr from cdist import core @@ -141,11 +144,42 @@ class Config(object): base_root_path = cls.create_base_root_path(args.out_path) hostcnt = 0 - for host in itertools.chain(cls.hosts(args.host), - cls.hosts(args.hostfile)): + + if args.tag or args.all_tagged_hosts: + inventory.determine_default_inventory_dir(args) + if args.all_tagged_hosts: + inv_list = inventory.InventoryList( + hosts=None, istag=True, hostfile=None, + db_basedir=args.inventory_dir) + else: + inv_list = inventory.InventoryList( + hosts=args.host, istag=True, hostfile=args.hostfile, + db_basedir=args.inventory_dir, + has_all_tags=args.has_all_tags) + it = inv_list.entries() + else: + it = itertools.chain(cls.hosts(args.host), + cls.hosts(args.hostfile)) + for entry in it: + if isinstance(entry, tuple): + # if configuring by specified tags + host = entry[0] + host_tags = entry[1] + else: + # if configuring by host then check inventory for tags + host = entry + inventory.determine_default_inventory_dir(args) + inv_list = inventory.InventoryList( + hosts=(host,), db_basedir=args.inventory_dir) + inv = tuple(inv_list.entries()) + if inv: + # host is present in inventory and has tags + host_tags = inv[0][1] + else: + # host is not present in inventory or has no tags + host_tags = None host_base_path, hostdir = cls.create_host_base_dirs( host, base_root_path) - log.debug("Base root path for target host \"{}\" is \"{}\"".format( host, host_base_path)) @@ -154,11 +188,12 @@ class Config(object): log.trace("Creating child process for %s", host) process[host] = multiprocessing.Process( target=cls.onehost, - args=(host, host_base_path, hostdir, args, True)) + args=(host, host_tags, host_base_path, hostdir, args, + True)) process[host].start() else: try: - cls.onehost(host, host_base_path, hostdir, + cls.onehost(host, host_tags, host_base_path, hostdir, args, parallel=False) except cdist.Error as e: failed_hosts.append(host) @@ -211,7 +246,8 @@ class Config(object): return (remote_exec, remote_copy, remote_cmds_cleanup, ) @classmethod - def onehost(cls, host, host_base_path, host_dir_name, args, parallel): + def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, + parallel): """Configure ONE system""" log = logging.getLogger(host) @@ -229,6 +265,7 @@ class Config(object): local = cdist.exec.local.Local( target_host=target_host, + target_host_tags=host_tags, base_root_path=host_base_path, host_dir_name=host_dir_name, initial_manifest=args.manifest, @@ -445,7 +482,7 @@ class Config(object): for chunk in cargo: for obj in chunk: if (obj.cdist_type == cdist_object.cdist_type and - cdist_object.cdist_type.is_nonparallel): + cdist_object.cdist_type.is_nonparallel): break else: chunk.append(cdist_object) diff --git a/cdist/core/code.py b/cdist/core/code.py index 1b5fe1d6..173d192d 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -56,6 +56,7 @@ gencode-local __object_fq: full qualified object id, iow: $type.name + / + object_id __type: full qualified path to the type's dir __files: full qualified path to the files dir + __target_host_tags: comma spearated list of host tags returns: string containing the generated code or None @@ -74,6 +75,7 @@ gencode-remote __object_fq: full qualified object id, iow: $type.name + / + object_id __type: full qualified path to the type's dir __files: full qualified path to the files dir + __target_host_tags: comma spearated list of host tags returns: string containing the generated code or None @@ -106,6 +108,7 @@ class Code(object): '__target_fqdn': self.target_host[2], '__global': self.local.base_path, '__files': self.local.files_path, + '__target_host_tags': self.local.target_host_tags, } def _run_gencode(self, cdist_object, which): diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index d604c015..38f2a921 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -77,6 +77,7 @@ class Explorer(object): '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__explorer': self.remote.global_explorer_path, + '__target_host_tags': self.local.target_host_tags, } self._type_explorers_transferred = [] self.jobs = jobs diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index d8570097..6f941550 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -42,6 +42,7 @@ common: types are defined for use in type emulator == local.type_path __files: full qualified path to the files dir + __target_host_tags: comma spearated list of host tags initial manifest is: script: full qualified path to the initial manifest @@ -109,6 +110,7 @@ class Manifest(object): '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__files': self.local.files_path, + '__target_host_tags': self.local.target_host_tags, } if self.log.getEffectiveLevel() == logging.DEBUG: diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 6c285204..23ad4ce9 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -49,6 +49,7 @@ class Local(object): """ def __init__(self, target_host, + target_host_tags, base_root_path, host_dir_name, exec_path=sys.argv[0], @@ -58,6 +59,10 @@ class Local(object): quiet_mode=False): self.target_host = target_host + if target_host_tags is None: + self.target_host_tags = "" + else: + self.target_host_tags = ",".join(target_host_tags) self.hostdir = host_dir_name self.base_path = os.path.join(base_root_path, "data") diff --git a/cdist/inventory.py b/cdist/inventory.py new file mode 100644 index 00000000..ccb4428f --- /dev/null +++ b/cdist/inventory.py @@ -0,0 +1,390 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import cdist +import logging +import os +import os.path +import itertools +import sys +from cdist.hostsource import hostfile_process_line + +DIST_INVENTORY_DB_NAME = "inventory" + +dist_inventory_db = os.path.abspath(os.path.join( + os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME)) + + +def determine_default_inventory_dir(args): + # The order of inventory dir setting by decreasing priority + # 1. inventory_dir argument + # 2. CDIST_INVENTORY_DIR env var if set + # 3. ~/.cdist/inventory if HOME env var is set + # 4. distribution inventory directory + if not args.inventory_dir: + if 'CDIST_INVENTORY_DIR' in os.environ: + args.inventory_dir = os.environ['CDIST_INVENTORY_DIR'] + else: + home = cdist.home_dir() + if home: + args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME) + else: + args.inventory_dir = dist_inventory_db + + +def contains_all(big, little): + """Return True if big contains all elements from little, + False otherwise. + """ + return set(little).issubset(set(big)) + + +def contains_any(big, little): + """Return True if big contains any element from little, + False otherwise. + """ + for x in little: + if x in big: + return True + return False + + +def check_always_true(x, y): + return True + + +def rstrip_nl(s): + '''str.rstrip "\n" from s''' + return str.rstrip(s, "\n") + + +class Inventory(object): + """Inventory main class""" + + def __init__(self, db_basedir=dist_inventory_db): + self.db_basedir = db_basedir + self.log = logging.getLogger("inventory") + self.init_db() + + def init_db(self): + self.log.debug("Init db: {}".format(self.db_basedir)) + if not os.path.exists(self.db_basedir): + os.makedirs(self.db_basedir, exist_ok=True) + elif not os.path.isdir(self.db_basedir): + raise cdist.Error(("Invalid inventory db basedir \'{}\'," + " must be a directory").format(self.db_basedir)) + + @staticmethod + def strlist_to_list(slist): + if slist: + result = [x for x in slist.split(',') if x] + else: + result = [] + return result + + def _input_values(self, source): + """Yield input values from source. + Source can be a sequence or filename (stdin if '-'). + In case of filename each line represents one input value. + """ + if isinstance(source, str): + import fileinput + try: + with fileinput.FileInput(files=(source)) as f: + for x in f: + result = hostfile_process_line(x, strip_func=rstrip_nl) + if result: + yield result + except (IOError, OSError) as e: + raise cdist.Error("Error reading from \'{}\'".format( + source)) + else: + if source: + for x in source: + if x: + yield x + + def _host_path(self, host): + hostpath = os.path.join(self.db_basedir, host) + return hostpath + + def _all_hosts(self): + return os.listdir(self.db_basedir) + + def _check_host(self, hostpath): + if not os.path.exists(hostpath): + return False + else: + if not os.path.isfile(hostpath): + raise cdist.Error(("Host path \'{}\' exists, but is not" + " a valid file").format(hostpath)) + return True + + def _read_host_tags(self, hostpath): + result = set() + with open(hostpath, "rt") as f: + for tag in f: + tag = tag.rstrip("\n") + if tag: + result.add(tag) + return result + + def _get_host_tags(self, host): + hostpath = self._host_path(host) + if self._check_host(hostpath): + return self._read_host_tags(hostpath) + else: + return None + + def _write_host_tags(self, host, tags): + hostpath = self._host_path(host) + if self._check_host(hostpath): + with open(hostpath, "wt") as f: + for tag in tags: + f.write("{}\n".format(tag)) + return True + else: + return False + + @classmethod + def commandline(cls, args): + """Manipulate inventory db""" + log = logging.getLogger("cdist") + if 'taglist' in args: + args.taglist = cls.strlist_to_list(args.taglist) + determine_default_inventory_dir(args) + + log.info("Using inventory: {}".format(args.inventory_dir)) + log.debug("Inventory args: {}".format(vars(args))) + log.debug("Inventory command: {}".format(args.subcommand)) + + if args.subcommand == "list": + c = InventoryList(hosts=args.host, istag=args.tag, + hostfile=args.hostfile, + db_basedir=args.inventory_dir, + list_only_host=args.list_only_host, + has_all_tags=args.has_all_tags) + elif args.subcommand == "add-host": + c = InventoryHost(hosts=args.host, hostfile=args.hostfile, + db_basedir=args.inventory_dir) + elif args.subcommand == "del-host": + c = InventoryHost(hosts=args.host, hostfile=args.hostfile, + all=args.all, db_basedir=args.inventory_dir, + action="del") + elif args.subcommand == "add-tag": + c = InventoryTag(hosts=args.host, tags=args.taglist, + hostfile=args.hostfile, tagfile=args.tagfile, + db_basedir=args.inventory_dir) + elif args.subcommand == "del-tag": + c = InventoryTag(hosts=args.host, tags=args.taglist, + hostfile=args.hostfile, tagfile=args.tagfile, + all=args.all, db_basedir=args.inventory_dir, + action="del") + else: + raise cdist.Error("Unknown inventory command \'{}\'".format( + args.subcommand)) + c.run() + + +class InventoryList(Inventory): + def __init__(self, hosts=None, istag=False, hostfile=None, + list_only_host=False, has_all_tags=False, + db_basedir=dist_inventory_db): + super().__init__(db_basedir) + self.hosts = hosts + self.istag = istag + self.hostfile = hostfile + self.list_only_host = list_only_host + self.has_all_tags = has_all_tags + + def _print(self, host, tags): + if self.list_only_host: + print("{}".format(host)) + else: + print("{} {}".format(host, ",".join(sorted(tags)))) + + def _do_list(self, it_tags, it_hosts, check_func): + if (it_tags is not None): + param_tags = set(it_tags) + self.log.debug("param_tags: {}".format(param_tags)) + else: + param_tags = set() + for host in it_hosts: + self.log.debug("host: {}".format(host)) + tags = self._get_host_tags(host) + if tags is None: + self.log.info("Host \'{}\' not found, skipped".format(host)) + continue + self.log.debug("tags: {}".format(tags)) + if check_func(tags, param_tags): + yield host, tags + + def entries(self): + if not self.hosts and not self.hostfile: + self.log.info("Listing all hosts") + it_hosts = self._all_hosts() + it_tags = None + check_func = check_always_true + else: + it = itertools.chain(self._input_values(self.hosts), + self._input_values(self.hostfile)) + if self.istag: + self.log.info("Listing by tag(s)") + it_hosts = self._all_hosts() + it_tags = it + if self.has_all_tags: + check_func = contains_all + else: + check_func = contains_any + else: + self.log.info("Listing by host(s)") + it_hosts = it + it_tags = None + check_func = check_always_true + for host, tags in self._do_list(it_tags, it_hosts, check_func): + yield host, tags + + def host_entries(self): + for host, tags in self.entries(): + yield host + + def run(self): + for host, tags in self.entries(): + self._print(host, tags) + + +class InventoryHost(Inventory): + def __init__(self, hosts=None, hostfile=None, + db_basedir=dist_inventory_db, all=False, action="add"): + super().__init__(db_basedir) + self.actions = ("add", "del") + if action not in self.actions: + raise cdist.Error("Invalid action \'{}\', valid actions are:" + " {}\n".format(action, self.actions.keys())) + self.action = action + self.hosts = hosts + self.hostfile = hostfile + self.all = all + + if not self.hosts and not self.hostfile: + self.hostfile = "-" + + def _new_hostpath(self, hostpath): + # create empty file + with open(hostpath, "w"): + pass + + def _action(self, host): + if self.action == "add": + self.log.info("Adding host \'{}\'".format(host)) + elif self.action == "del": + self.log.info("Deleting host \'{}\'".format(host)) + hostpath = self._host_path(host) + self.log.debug("hostpath: {}".format(hostpath)) + if self.action == "add" and not os.path.exists(hostpath): + self._new_hostpath(hostpath) + else: + if not os.path.isfile(hostpath): + raise cdist.Error(("Host path \'{}\' is" + " not a valid file").format(hostpath)) + if self.action == "del": + os.remove(hostpath) + + def run(self): + if self.action == "del" and self.all: + self.log.debug("Doing for all hosts") + it = self._all_hosts() + else: + self.log.debug("Doing for specified hosts") + it = itertools.chain(self._input_values(self.hosts), + self._input_values(self.hostfile)) + for host in it: + self._action(host) + + +class InventoryTag(Inventory): + def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None, + db_basedir=dist_inventory_db, all=False, action="add"): + super().__init__(db_basedir) + self.actions = ("add", "del") + if action not in self.actions: + raise cdist.Error("Invalid action \'{}\', valid actions are:" + " {}\n".format(action, self.actions.keys())) + self.action = action + self.hosts = hosts + self.tags = tags + self.hostfile = hostfile + self.tagfile = tagfile + self.all = all + + if not self.hosts and not self.hostfile: + self.allhosts = True + else: + self.allhosts = False + if not self.tags and not self.tagfile: + self.tagfile = "-" + + if self.hostfile == "-" and self.tagfile == "-": + raise cdist.Error("Cannot read both, hosts and tags, from stdin") + + def _read_input_tags(self): + self.input_tags = set() + for tag in itertools.chain(self._input_values(self.tags), + self._input_values(self.tagfile)): + self.input_tags.add(tag) + + def _action(self, host): + host_tags = self._get_host_tags(host) + if host_tags is None: + print("Host \'{}\' does not exist, skipping".format(host), + file=sys.stderr) + return + self.log.debug("existing host_tags: {}".format(host_tags)) + if self.action == "del" and self.all: + host_tags = set() + else: + for tag in self.input_tags: + if self.action == "add": + self.log.info("Adding tag \'{}\' for host \'{}\'".format( + tag, host)) + host_tags.add(tag) + elif self.action == "del": + self.log.info("Deleting tag \'{}\' for host \'{}\'".format( + tag, host)) + if tag in host_tags: + host_tags.remove(tag) + self.log.debug("new host tags: {}".format(host_tags)) + if not self._write_host_tags(host, host_tags): + self.log.info("{} does not exist, skipped".format(host)) + + def run(self): + if self.allhosts: + self.log.debug("Doing for all hosts") + it = self._all_hosts() + else: + self.log.debug("Doing for specified hosts") + it = itertools.chain(self._input_values(self.hosts), + self._input_values(self.hostfile)) + if not(self.action == "del" and self.all): + self._read_input_tags() + for host in it: + self._action(host) diff --git a/cdist/shell.py b/cdist/shell.py index 662f8f7d..44cdd49d 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -44,6 +44,7 @@ class Shell(object): "cdist-shell-no-target-host", "cdist-shell-no-target-host", ) + self.target_host_tags = "" host_dir_name = cdist.str_hash(self.target_host[0]) base_root_path = tempfile.mkdtemp() @@ -51,6 +52,7 @@ class Shell(object): self.local = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=host_dir_name) @@ -77,6 +79,7 @@ class Shell(object): '__manifest': self.local.manifest_path, '__explorer': self.local.global_explorer_path, '__files': self.local.files_path, + '__target_host_tags': self.local.target_host_tags, } self.env.update(additional_env) diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py index 83b0c618..faa3686a 100644 --- a/cdist/test/__init__.py +++ b/cdist/test/__init__.py @@ -42,6 +42,7 @@ class CdistTestCase(unittest.TestCase): 'cdisttesthost', 'cdisttesthost', ) + target_host_tags = "tag1,tag2,tag3" def mkdtemp(self, **kwargs): return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs) diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 83c93f8b..50da2b8a 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -46,6 +46,7 @@ class CodeTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=cdist.test.cdist_exec_path, @@ -97,6 +98,8 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.object_id) self.assertEqual(output_dict['__object_name'], self.cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) def test_run_gencode_remote_environment(self): output_string = self.code.run_gencode_remote(self.cdist_object) @@ -120,6 +123,8 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.object_id) self.assertEqual(output_dict['__object_name'], self.cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) def test_transfer_code_remote(self): self.cdist_object.code_remote = self.code.run_gencode_remote( diff --git a/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local b/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local index 7fa70342..56744a27 100755 --- a/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local +++ b/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local @@ -9,3 +9,4 @@ echo "echo __object: $__object" echo "echo __object_id: $__object_id" echo "echo __object_name: $__object_name" echo "echo __files: $__files" +echo "echo __target_host_tags: $__target_host_tags" diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index af1aa38f..5ff98269 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -60,6 +60,7 @@ class ConfigRunTestCase(test.CdistTestCase): os.makedirs(self.host_base_path) self.local = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir) @@ -164,6 +165,7 @@ class ConfigRunTestCase(test.CdistTestCase): """Test if the dryrun option is working like expected""" drylocal = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir, # exec_path can not derivated from sys.argv in case of unittest @@ -181,6 +183,7 @@ class ConfigRunTestCase(test.CdistTestCase): """Test to show dependency resolver warning message.""" local = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=os.path.abspath(os.path.join( diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 51de3180..664ab20b 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -53,6 +53,7 @@ class EmulatorTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -156,6 +157,7 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -246,6 +248,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -279,6 +282,7 @@ class OverrideTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -322,6 +326,7 @@ class ArgumentsTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -445,6 +450,7 @@ class StdinTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -511,6 +517,7 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py index fc66020d..928b4e0d 100644 --- a/cdist/test/explorer/__init__.py +++ b/cdist/test/explorer/__init__.py @@ -50,6 +50,7 @@ class ExplorerClassTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=base_root_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py new file mode 100644 index 00000000..4c0dd936 --- /dev/null +++ b/cdist/test/inventory/__init__.py @@ -0,0 +1,476 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import shutil +import cdist +import os.path as op +import unittest +import sys +from cdist import test +from cdist import inventory +from io import StringIO + +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') +inventory_dir = op.join(fixtures, "inventory") + + +class InventoryTestCase(test.CdistTestCase): + + def _create_host_with_tags(self, host, tags): + os.makedirs(inventory_dir, exist_ok=True) + hostfile = op.join(inventory_dir, host) + with open(hostfile, "w") as f: + for x in tags: + f.write("{}\n".format(x)) + + def setUp(self): + self.maxDiff = None + self.db = { + "loadbalancer1": ["loadbalancer", "all", "europe", ], + "loadbalancer2": ["loadbalancer", "all", "europe", ], + "loadbalancer3": ["loadbalancer", "all", "africa", ], + "loadbalancer4": ["loadbalancer", "all", "africa", ], + "web1": ["web", "all", "static", ], + "web2": ["web", "all", "dynamic", ], + "web3": ["web", "all", "dynamic", ], + "shell1": ["shell", "all", "free", ], + "shell2": ["shell", "all", "free", ], + "shell3": ["shell", "all", "charge", ], + "shell4": ["shell", "all", "charge", ], + "monty": ["web", "python", "shell", ], + "python": ["web", "python", "shell", ], + } + for x in self.db: + self.db[x] = sorted(self.db[x]) + for host in self.db: + self._create_host_with_tags(host, self.db[host]) + self.sys_stdout = sys.stdout + out = StringIO() + sys.stdout = out + + def _get_output(self): + sys.stdout.flush() + output = sys.stdout.getvalue().strip() + return output + + def tearDown(self): + sys.stdout = self.sys_stdout + shutil.rmtree(inventory_dir) + + def test_inventory_create_db(self): + dbdir = op.join(fixtures, "foo") + inv = inventory.Inventory(db_basedir=dbdir) + self.assertTrue(os.path.isdir(dbdir)) + self.assertEqual(inv.db_basedir, dbdir) + shutil.rmtree(inv.db_basedir) + + # InventoryList + def test_inventory_list_print(self): + invList = inventory.InventoryList(db_basedir=inventory_dir) + invList.run() + output = self._get_output() + self.assertTrue(' ' in output) + + def test_inventory_list_print_host_only(self): + invList = inventory.InventoryList(db_basedir=inventory_dir, + list_only_host=True) + invList.run() + output = self._get_output() + self.assertFalse(' ' in output) + + def test_inventory_list_all(self): + invList = inventory.InventoryList(db_basedir=inventory_dir) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + self.assertEqual(db, self.db) + + def test_inventory_list_by_host_hosts(self): + hosts = ("web1", "web2", "web3",) + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = {host: sorted(self.db[host]) for host in hosts} + self.assertEqual(db, expected_db) + + def test_inventory_list_by_host_hostfile(self): + hosts = ("web1", "web2", "web3",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hosts: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + hostfile=hostfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = {host: sorted(self.db[host]) for host in hosts} + self.assertEqual(db, expected_db) + os.remove(hostfile) + + def test_inventory_list_by_host_hosts_hostfile(self): + hosts = ("shell1", "shell4",) + hostsf = ("web1", "web2", "web3",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts, hostfile=hostfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + import itertools + expected_db = {host: sorted(self.db[host]) for host in + itertools.chain(hostsf, hosts)} + self.assertEqual(db, expected_db) + os.remove(hostfile) + + def _gen_expected_db_for_tags(self, tags): + db = {} + for host in self.db: + for tag in tags: + if tag in self.db[host]: + db[host] = self.db[host] + break + return db + + def _gen_expected_db_for_has_all_tags(self, tags): + db = {} + for host in self.db: + if set(tags).issubset(set(self.db[host])): + db[host] = self.db[host] + return db + + def test_inventory_list_by_tag_hosts(self): + tags = ("web", "shell",) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hosts=tags) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = self._gen_expected_db_for_tags(tags) + self.assertEqual(db, expected_db) + + def test_inventory_list_by_tag_hostfile(self): + tags = ("web", "shell",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tags: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hostfile=tagfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = self._gen_expected_db_for_tags(tags) + self.assertEqual(db, expected_db) + os.remove(tagfile) + + def test_inventory_list_by_tag_hosts_hostfile(self): + tags = ("web", "shell",) + tagsf = ("dynamic", "europe",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hosts=tags, + hostfile=tagfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + import itertools + expected_db = self._gen_expected_db_for_tags(tags + tagsf) + self.assertEqual(db, expected_db) + os.remove(tagfile) + + def test_inventory_list_by_tag_has_all_tags(self): + tags = ("web", "python", "shell",) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hosts=tags, + has_all_tags=True) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = self._gen_expected_db_for_has_all_tags(tags) + self.assertEqual(db, expected_db) + + # InventoryHost + def test_inventory_host_add_hosts(self): + hosts = ("spam", "eggs", "foo",) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="add", hosts=hosts) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + expected_hosts = tuple(x for x in invList.host_entries() if x in hosts) + self.assertEqual(sorted(hosts), sorted(expected_hosts)) + + def test_inventory_host_add_hostfile(self): + hosts = ("spam-new", "eggs-new", "foo-new",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hosts: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="add", hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + expected_hosts = tuple(x for x in invList.host_entries() if x in hosts) + self.assertEqual(sorted(hosts), sorted(expected_hosts)) + os.remove(hostfile) + + def test_inventory_host_add_hosts_hostfile(self): + hosts = ("spam-spam", "eggs-spam", "foo-spam",) + hostf = ("spam-eggs-spam", "spam-foo-spam",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostf: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="add", hosts=hosts, + hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostf) + expected_hosts = tuple(invList.host_entries()) + self.assertEqual(sorted(hosts + hostf), sorted(expected_hosts)) + os.remove(hostfile) + + def test_inventory_host_del_hosts(self): + hosts = ("web1", "shell1",) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hosts=hosts) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts) + expected_hosts = tuple(invList.host_entries()) + self.assertTupleEqual(expected_hosts, ()) + + def test_inventory_host_del_hostfile(self): + hosts = ("loadbalancer3", "loadbalancer4",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hosts: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts) + expected_hosts = tuple(invList.host_entries()) + self.assertTupleEqual(expected_hosts, ()) + os.remove(hostfile) + + def test_inventory_host_del_hosts_hostfile(self): + hosts = ("loadbalancer1", "loadbalancer2",) + hostf = ("web2", "shell2",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostf: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hosts=hosts, + hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostf) + expected_hosts = tuple(invList.host_entries()) + self.assertTupleEqual(expected_hosts, ()) + os.remove(hostfile) + + @unittest.expectedFailure + def test_inventory_host_invalid_host(self): + try: + invalid_hostfile = op.join(inventory_dir, "invalid") + os.mkdir(invalid_hostfile) + hosts = ("invalid",) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hosts=hosts) + invHost.run() + except e: + os.rmdir(invalid_hostfile) + raise e + + # InventoryTag + def test_inventory_tag_init(self): + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add") + self.assertTrue(invTag.allhosts) + self.assertEqual(invTag.tagfile, "-") + + def test_inventory_tag_stdin_multiple_hosts(self): + try: + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tagfile="-", + hosts=("host1", "host2",)) + except e: + self.fail() + + def test_inventory_tag_stdin_hostfile(self): + try: + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tagfile="-", + hostfile="hosts") + except e: + self.fail() + + @unittest.expectedFailure + def test_inventory_tag_stdin_both(self): + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tagfile="-", + hostfile="-") + + def test_inventory_tag_add_for_all_hosts(self): + tags = ("spam-spam-spam", "spam-spam-eggs",) + tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tags=tags, + tagfile=tagfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + failed = False + for host, taglist in invList.entries(): + for x in tagsf + tags: + if x not in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + if failed: + self.fail() + + def test_inventory_tag_add(self): + tags = ("spam-spam-spam", "spam-spam-eggs",) + tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",) + hosts = ("loadbalancer1", "loadbalancer2", "shell2",) + hostsf = ("web2", "web3",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tags=tags, + tagfile=tagfile, hosts=hosts, + hostfile=hostfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostsf) + failed = False + for host, taglist in invList.entries(): + if host not in hosts + hostsf: + failed = True + break + for x in tagsf + tags: + if x not in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + os.remove(hostfile) + if failed: + self.fail() + + def test_inventory_tag_del_for_all_hosts(self): + tags = ("all",) + tagsf = ("charge",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="del", tags=tags, + tagfile=tagfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + failed = False + for host, taglist in invList.entries(): + for x in tagsf + tags: + if x in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + if failed: + self.fail() + + def test_inventory_tag_del(self): + tags = ("europe", "africa",) + tagsf = ("free", ) + hosts = ("loadbalancer1", "loadbalancer2", "shell2",) + hostsf = ("web2", "web3",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="del", tags=tags, + tagfile=tagfile, hosts=hosts, + hostfile=hostfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostsf) + failed = False + for host, taglist in invList.entries(): + if host not in hosts + hostsf: + failed = True + break + for x in tagsf + tags: + if x in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + os.remove(hostfile) + if failed: + self.fail() + + def test_inventory_tag_del_all_tags(self): + hosts = ("web3", "shell1",) + hostsf = ("shell2", "loadbalancer1",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", all=True, + hosts=hosts, hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostsf) + for host, htags in invList.entries(): + self.assertEqual(htags, ()) + os.remove(hostfile) + +if __name__ == "__main__": + unittest.main() diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index 3e07c1a7..e0da2d9f 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -53,6 +53,7 @@ class ManifestTestCase(test.CdistTestCase): base_root_path = os.path.join(out_path, hostdir) self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=base_root_path, host_dir_name=hostdir, exec_path=cdist.test.cdist_exec_path, @@ -93,6 +94,8 @@ class ManifestTestCase(test.CdistTestCase): self.local.type_path) self.assertEqual(output_dict['__manifest'], self.local.manifest_path) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) def test_type_manifest_environment(self): cdist_type = core.CdistType(self.local.type_path, '__dump_environment') @@ -126,6 +129,8 @@ class ManifestTestCase(test.CdistTestCase): self.assertEqual(output_dict['__object_id'], cdist_object.object_id) self.assertEqual(output_dict['__object_name'], cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) def test_debug_env_setup(self): current_level = self.log.getEffectiveLevel() diff --git a/cdist/test/manifest/fixtures/conf/manifest/dump_environment b/cdist/test/manifest/fixtures/conf/manifest/dump_environment index 702145e2..9f9df372 100755 --- a/cdist/test/manifest/fixtures/conf/manifest/dump_environment +++ b/cdist/test/manifest/fixtures/conf/manifest/dump_environment @@ -9,4 +9,5 @@ __global: $__global __cdist_type_base_path: $__cdist_type_base_path __manifest: $__manifest __files: $__files +__target_host_tags: $__target_host_tags DONE diff --git a/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest b/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest index 757d07b5..fec5cb3f 100755 --- a/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest +++ b/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest @@ -13,4 +13,5 @@ __object: $__object __object_id: $__object_id __object_name: $__object_name __files: $__files +__target_host_tags: $__target_host_tags DONE diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index 6b58e2a2..1311384a 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -5,8 +5,8 @@ _cdist() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}" - opts="-h --help -d --debug -v --verbose -V --version" - cmds="banner shell config install" + opts="-h --help -q --quiet -v --verbose -V --version" + cmds="banner config install inventory shell" case "${prevprev}" in shell) @@ -18,6 +18,41 @@ _cdist() ;; esac ;; + inventory) + case "${prev}" in + list) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --invento/y -a --all -f --file -H --host-only \ + -t --tag" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + add-host) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -f --file" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + del-host) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -a --all -f --file" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + add-tag) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -f --file -T --tag-file -t --taglist" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + del-tag) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -a --all -f --file -T --tag-file -t --taglist" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + esac + ;; esac case "${prev}" in @@ -26,23 +61,31 @@ _cdist() return 0 ;; banner) - opts="-h --help -d --debug -v --verbose" + opts="-h --help -q --quiet -v --verbose" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; shell) - opts="-h --help -d --debug -v --verbose -s --shell" + opts="-h --help -q --quiet -v --verbose -s --shell" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; config|install) - opts="-h --help -d --debug -v --verbose -b --beta \ - -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \ - -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential \ - --remote-copy --remote-exec" + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -C --cache-path-pattern -c --conf-dir \ + -f --file -i --initial-manifest -A --all-tagged \ + -j --jobs -n --dry-run -o --out-dir -p --parallel \ + -r --remote-out-dir \ + -s --sequential --remote-copy --remote-exec -t --tag -a --all" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + inventory) + cmds="list add-host del-host add-tag del-tag" + opts="-h --help -q --quiet -v --verbose" + COMPREPLY=( $(compgen -W "${opts} ${cmds}" -- ${cur}) ) + return 0 + ;; *) ;; esac diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 082cad12..405023ed 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -11,16 +11,16 @@ _cdist() case $state in opts_cmds) - _arguments '1:Options and commands:(banner config shell install -h --help -d --debug -v --verbose -V --version)' + _arguments '1:Options and commands:(banner config install inventory shell -h --help -q --quiet -v --verbose -V --version)' ;; *) case $words[2] in -*) - opts=(-h --help -d --debug -v --verbose -V --version) + opts=(-h --help -q --quiet -v --verbose -V --version) compadd "$@" -- $opts ;; banner) - opts=(-h --help -d --debug -v --verbose) + opts=(-h --help -q --quiet -v --verbose) compadd "$@" -- $opts ;; shell) @@ -30,16 +30,45 @@ _cdist() compadd "$@" -- $shells ;; *) - opts=(-h --help -d --debug -v --verbose -s --shell) + opts=(-h --help -q --quiet -v --verbose -s --shell) compadd "$@" -- $opts ;; esac ;; config|install) - opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec) + opts=(-h --help -q --quiet -v --verbose -a --all -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec -t --tag -I --inventory -A --all-tagged) compadd "$@" -- $opts ;; - *) + inventory) + case $words[3] in + list) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -H --host-only -t --tag) + compadd "$@" -- $opts + ;; + add-host) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file) + compadd "$@" -- $opts + ;; + del-host) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file) + compadd "$@" -- $opts + ;; + add-tag) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file -T --tag-file -t --taglist) + compadd "$@" -- $opts + ;; + del-tag) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -T --tag-file -t --taglist) + compadd "$@" -- $opts + ;; + *) + cmds=(list add-host del-host add-tag del-tag) + opts=(-h --help -q --quiet -v --verbose) + compadd "$@" -- $cmds $opts + ;; + esac + ;; + *) ;; esac esac diff --git a/docs/changelog b/docs/changelog index fee1e1ff..f4bfaffb 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,11 @@ Changelog --------- next: + * Core: Add inventory functionality (Darko Poljak) + * Core: Expose inventory host tags in __target_host_tags env var (Darko Poljak) + * Type __timezone: Check current timezone before doing anything (Ander Punnar) + +4.5.0: 2017-07-20 * Types: Fix install types (Steven Armstrong) * Core: Add -r command line option for setting remote base path (Steven Armstrong) * Core: Allow manifest and gencode scripts to be written in any language (Darko Poljak) diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst index fdbb7a80..45aba11b 100644 --- a/docs/src/cdist-best-practice.rst +++ b/docs/src/cdist-best-practice.rst @@ -97,7 +97,7 @@ Including a possible common base that is reused across the different sites:: git merge common -The following **.git/config** is taken from a real world scenario: +The following **.git/config** is taken from a real world scenario:: # Track upstream, merge from time to time [remote "upstream"] diff --git a/docs/src/cdist-inventory.rst b/docs/src/cdist-inventory.rst new file mode 100644 index 00000000..584fe310 --- /dev/null +++ b/docs/src/cdist-inventory.rst @@ -0,0 +1,211 @@ +Inventory +========= + +Introduction +------------ + +cdist comes with simple built-in tag based inventory. It is a simple inventory +with list of hosts and a host has a list of tags. +Inventory functionality is still in **beta** so it can be used only if beta +command line flag is specified (-b, --beta) or setting CDIST_BETA env var. + +Description +----------- + +The idea is to have simple tagging inventory. There is a list of hosts and for +each host there are tags. Inventory database is a set of files under inventory +database base directory. Filename equals hostname. Each file contains tags for +hostname with each tag on its own line. + +Using inventory you can now configure hosts by selecting them by tags. + +Tags have no values, as tags are just tags. Tag name-value would in this +context mean that host has two tags and it is selected by specifying that both +tags are present. + +This inventory is **KISS** cdist built-in inventory database. You can maintain it +using cdist inventory interface or using standard UNIX tools. + +cdist inventory interface +------------------------- + +With cdist inventory interface you can list host(s) and tag(s), add host(s), +add tag(s), delete host(s) and delete tag(s). + +Configuring hosts using inventory +--------------------------------- + +config command now has new options, **-t**, **-a** and **-A**. + +**-A** means that all hosts in tag db is selected. + +**-a** means that selected hosts must contain ALL specified tags. + +**-t** means that host specifies tag - all hosts that have specified tags are +selected. + +Examples +-------- + +.. code-block:: sh + + # List inventory content + $ cdist inventory list -b + + # List inventory for specified host localhost + $ cdist inventory list -b localhost + + # List inventory for specified tag loadbalancer + $ cdist inventory list -b -t loadbalancer + + # Add hosts to inventory + $ cdist inventory add-host -b web1 web2 web3 + + # Delete hosts from file old-hosts from inventory + $ cdist inventory del-host -b -f old-hosts + + # Add tags to specifed hosts + $ cdist inventory add-tag -b -t europe,croatia,web,static web1 web2 + + # Add tag to all hosts in inventory + $ cdist inventory add-tag -b -t vm + + # Delete all tags from specified host + $ cdist inventory del-tag -b -a localhost + + # Delete tags read from stdin from hosts specified by file hosts + $ cdist inventory del-tag -b -T - -f hosts + + # Configure hosts from inventory with any of specified tags + $ cdist config -b -t web dynamic + + # Configure hosts from inventory with all specified tags + $ cdist config -b -t -a web dynamic + + # Configure all hosts from inventory db + $ cdist config -b -A + +Example of manipulating database +-------------------------------- + +.. code-block:: sh + + $ python3 scripts/cdist inventory list -b + $ python3 scripts/cdist inventory add-host -b localhost + $ python3 scripts/cdist inventory add-host -b test.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost + test.mycloud.net + $ python3 scripts/cdist inventory add-host -b web1.mycloud.net web2.mycloud.net shell1.mycloud.net shell2.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost + test.mycloud.net + web1.mycloud.net + web2.mycloud.net + shell1.mycloud.net + shell2.mycloud.net + $ python3 scripts/cdist inventory add-tag -b -t web web1.mycloud.net web2.mycloud.net + $ python3 scripts/cdist inventory add-tag -b -t shell shell1.mycloud.net shell2.mycloud.net + $ python3 scripts/cdist inventory add-tag -b -t cloud + $ python3 scripts/cdist inventory list -b + localhost cloud + test.mycloud.net cloud + web1.mycloud.net cloud,web + web2.mycloud.net cloud,web + shell1.mycloud.net cloud,shell + shell2.mycloud.net cloud,shell + $ python3 scripts/cdist inventory add-tag -b -t test,web,shell test.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost cloud + test.mycloud.net cloud,shell,test,web + web1.mycloud.net cloud,web + web2.mycloud.net cloud,web + shell1.mycloud.net cloud,shell + shell2.mycloud.net cloud,shell + $ python3 scripts/cdist inventory del-tag -b -t shell test.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost cloud + test.mycloud.net cloud,test,web + web1.mycloud.net cloud,web + web2.mycloud.net cloud,web + shell1.mycloud.net cloud,shell + shell2.mycloud.net cloud,shell + $ python3 scripts/cdist inventory add-tag -b -t all + $ python3 scripts/cdist inventory add-tag -b -t mistake + $ python3 scripts/cdist inventory list -b + localhost all,cloud,mistake + test.mycloud.net all,cloud,mistake,test,web + web1.mycloud.net all,cloud,mistake,web + web2.mycloud.net all,cloud,mistake,web + shell1.mycloud.net all,cloud,mistake,shell + shell2.mycloud.net all,cloud,mistake,shell + $ python3 scripts/cdist inventory del-tag -b -t mistake + $ python3 scripts/cdist inventory list -b + localhost all,cloud + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + shell1.mycloud.net all,cloud,shell + shell2.mycloud.net all,cloud,shell + $ python3 scripts/cdist inventory del-host -b localhost + $ python3 scripts/cdist inventory list -b + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + shell1.mycloud.net all,cloud,shell + shell2.mycloud.net all,cloud,shell + $ python3 scripts/cdist inventory list -b -t web + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + $ python3 scripts/cdist inventory list -b -t -a web test + test.mycloud.net all,cloud,test,web + $ python3 scripts/cdist inventory list -b -t -a web all + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + $ python3 scripts/cdist inventory list -b -t web all + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + shell1.mycloud.net all,cloud,shell + shell2.mycloud.net all,cloud,shell + $ cd cdist/inventory + $ ls -1 + shell1.mycloud.net + shell2.mycloud.net + test.mycloud.net + web1.mycloud.net + web2.mycloud.net + $ ls -l + total 20 + -rw-r--r-- 1 darko darko 16 Jun 24 12:43 shell1.mycloud.net + -rw-r--r-- 1 darko darko 16 Jun 24 12:43 shell2.mycloud.net + -rw-r--r-- 1 darko darko 19 Jun 24 12:43 test.mycloud.net + -rw-r--r-- 1 darko darko 14 Jun 24 12:43 web1.mycloud.net + -rw-r--r-- 1 darko darko 14 Jun 24 12:43 web2.mycloud.net + $ cat test.mycloud.net + test + all + web + cloud + $ cat web2.mycloud.net + all + web + cloud + +For more info about inventory commands and options see `cdist `_\ (1). + +Using external inventory +------------------------ + +cdist can be used with any external inventory where external inventory is +some storage or database from which you can get a list of hosts to configure. +cdist can then be fed with this list of hosts through stdin or file using +**-f** option. For example, if your host list is stored in sqlite3 database +hosts.db and you want to select hosts which purpose is **django** then you +can use it with cdist like: + +.. code-block:: sh + + $ sqlite3 hosts.db "select hostname from hosts where purpose = 'django';" | cdist config diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 5889ded9..89820358 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -63,6 +63,10 @@ cdist/conf/ The distribution configuration directory. This contains types and explorers to be used. +cdist/inventory/ + The distribution inventory directory. + This path is relative to cdist installation directory. + confdir Cdist will use all available configuration directories and create a temporary confdir containing links to the real configuration directories. @@ -239,6 +243,9 @@ __target_fqdn This variable is derived from **__target_host** (using **socket.getfqdn()**). Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. +__target_host_tags + Comma separated list of target host tags. + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __type Path to the current type. Available for: type manifest, type gencode. @@ -274,6 +281,9 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). +CDIST_INVENTORY_DIR + Use this directory as inventory directory. + CDIST_BETA Enable beta functionalities. diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 59423332..bfe3c35d 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -64,15 +64,23 @@ If a type is flagged with 'install' flag then it is used only with install comma With other commands, i.e. config, these types are skipped if used. +Nonparallel types +----------------- +If a type is flagged with 'nonparallel' flag then its objects cannot be run in parallel +when using -j option. Example of such a type is __package_dpkg type where dpkg itself +prevents to be run in more than one instance. + + How to write a new type ----------------------- A type consists of -- parameter (optional) -- manifest (optional) -- singleton (optional) -- explorer (optional) -- gencode (optional) +- parameter (optional) +- manifest (optional) +- singleton (optional) +- explorer (optional) +- gencode (optional) +- nonparallel (optional) Types are stored below cdist/conf/type/. Their name should always be prefixed with two underscores (__) to prevent collisions with other executables in $PATH. @@ -240,6 +248,19 @@ install: create the (empty) file "install" in your type directory: With other commands, i.e. config, it will be skipped if used. +Nonparallel - only one instance can be run at a time +---------------------------------------------------- +If objects of a type must not or cannot be run in parallel when using -j +option, you must mark it as nonparallel: create the (empty) file "nonparallel" +in your type directory: + +.. code-block:: sh + + touch cdist/conf/type/__NAME/nonparallel + +For example, package types are nonparallel types. + + The type explorers ------------------ If a type needs to explore specific details, it can provide type specific diff --git a/docs/src/index.rst b/docs/src/index.rst index b33b707d..42c21199 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -24,6 +24,7 @@ Contents: cdist-explorer cdist-messaging cdist-parallelization + cdist-inventory cdist-reference cdist-best-practice cdist-stages diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index edcad828..d6bd1c8f 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,23 +11,48 @@ SYNOPSIS :: - cdist [-h] [-v] [-V] {banner,config,shell,install} ... + cdist [-h] [-q] [-v] [-V] {banner,config,install,inventory,shell} ... - cdist banner [-h] [-v] + cdist banner [-h] [-q] [-v] - cdist config [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + cdist config [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s] - [host [host ...]] + [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a] + [-f HOSTFILE] [-p] [-s] [-t] + [host [host ...]] - cdist install [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + cdist install [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] - [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] - [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s] - [host [host ...]] + [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY] + [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a] + [-f HOSTFILE] [-p] [-s] [-t] + [host [host ...]] - cdist shell [-h] [-v] [-s SHELL] + cdist inventory [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] + {add-host,add-tag,del-host,del-tag,list} ... + + cdist inventory add-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] + [-f HOSTFILE] + [host [host ...]] + + cdist inventory add-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] + [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [host [host ...]] + + cdist inventory del-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] + [host [host ...]] + + cdist inventory del-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [host [host ...]] + + cdist inventory list [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] [-H] [-t] + [host [host ...]] + + cdist shell [-h] [-q] [-v] [-s SHELL] DESCRIPTION @@ -72,6 +97,15 @@ CONFIG/INSTALL -------------- Configure/install one or more hosts. +.. option:: -A, --all-tagged + + use all hosts present in tags db + +.. option:: -a, --all + + list hosts that have all specified tags, if -t/--tag + is specified + .. option:: -b, --beta Enable beta functionality. @@ -103,6 +137,16 @@ Configure/install one or more hosts. read hosts from stdin. For the file format see :strong:`HOSTFILE FORMAT` below. +.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR + + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if this + argument is set then specified directory is used, if + CDIST_INVENTORY_DIR env var is set then its value is + used, if HOME env var is set then ~/.cdit/inventory is + used, otherwise distribution inventory directory is + used. + .. option:: -i MANIFEST, --initial-manifest MANIFEST Path to a cdist manifest or - to read from stdin @@ -125,7 +169,7 @@ Configure/install one or more hosts. Operate on multiple hosts in parallel -.. option:: -r, --remote-out-dir +.. option:: -r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH Directory to save cdist output in on the target host @@ -141,6 +185,10 @@ Configure/install one or more hosts. Command to use for remote execution (should behave like ssh) +.. option:: -t, --tag + + host is specified by tag, not hostname/address; list + all hosts that contain any of specified tags HOSTFILE FORMAT ~~~~~~~~~~~~~~~ @@ -174,6 +222,246 @@ Resulting path is used to specify cache path subdirectory under which current host cache data are saved. +INVENTORY +--------- +Manage inventory database. +Currently in beta with all sub-commands. + + +INVENTORY ADD-HOST +------------------ +Add host(s) to inventory database. + +.. option:: host + + host(s) to add + +.. option:: -b, --beta + + Enable beta functionalities. Beta functionalities + include inventory command with all sub-commands and + all options; config sub-command options: -j/--jobs, + -t/--tag, -a/--all. + + Can also be enabled using CDIST_BETA env var. + +.. option:: -f HOSTFILE, --file HOSTFILE + + Read additional hosts to add from specified file or + from stdin if '-' (each host on separate line). If no + host or host file is specified then, by default, read + from stdin. Hostfile format is the same as config hostfile format. + +.. option:: -h, --help + + show this help message and exit + +.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR + + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if this + argument is set then specified directory is used, if + CDIST_INVENTORY_DIR env var is set then its value is + used, if HOME env var is set then ~/.cdist/inventory is + used, otherwise distribution inventory directory is + used. + + +INVENTORY ADD-TAG +----------------- +Add tag(s) to inventory database. + +.. option:: host + + list of host(s) for which tags are added + +.. option:: -b, --beta + + Enable beta functionalities. Beta functionalities + include inventory command with all sub-commands and + all options; config sub-command options: -j/--jobs, + -t/--tag, -a/--all. + + Can also be enabled using CDIST_BETA env var. + +.. option:: -f HOSTFILE, --file HOSTFILE + + Read additional hosts to add tags from specified file + or from stdin if '-' (each host on separate line). If + no host or host file is specified then, by default, + read from stdin. If no tags/tagfile nor hosts/hostfile + are specified then tags are read from stdin and are + added to all hosts. Hostfile format is the same as config hostfile format. + +.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR + + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if this + argument is set then specified directory is used, if + CDIST_INVENTORY_DIR env var is set then its value is + used, if HOME env var is set then ~/.cdist/inventory is + used, otherwise distribution inventory directory is + used. + +.. option:: -T TAGFILE, --tag-file TAGFILE + + Read additional tags to add from specified file or + from stdin if '-' (each tag on separate line). If no + tag or tag file is specified then, by default, read + from stdin. If no tags/tagfile nor hosts/hostfile are + specified then tags are read from stdin and are added + to all hosts. Tagfile format is the same as config hostfile format. + +.. option:: -t TAGLIST, --taglist TAGLIST + + Tag list to be added for specified host(s), comma + separated values + + +INVENTORY DEL-HOST +------------------ +Delete host(s) from inventory database. + +.. option:: host + + host(s) to delete + +.. option:: -a, --all + + Delete all hosts + +.. option:: -b, --beta + + Enable beta functionalities. Beta functionalities + include inventory command with all sub-commands and + all options; config sub-command options: -j/--jobs, + -t/--tag, -a/--all. + + Can also be enabled using CDIST_BETA env var. + +.. option:: -f HOSTFILE, --file HOSTFILE + + Read additional hosts to delete from specified file or + from stdin if '-' (each host on separate line). If no + host or host file is specified then, by default, read + from stdin. Hostfile format is the same as config hostfile format. + +.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR + + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if this + argument is set then specified directory is used, if + CDIST_INVENTORY_DIR env var is set then its value is + used, if HOME env var is set then ~/.cdist/inventory is + used, otherwise distribution inventory directory is + used. + + +INVENTORY DEL-TAG +----------------- +Delete tag(s) from inventory database. + +.. option:: host + + list of host(s) for which tags are deleted + +.. option:: -a, --all + + Delete all tags for specified host(s) + +.. option:: -b, --beta + + Enable beta functionalities. Beta functionalities + include inventory command with all sub-commands and + all options; config sub-command options: -j/--jobs, + -t/--tag, -a/--all. + + Can also be enabled using CDIST_BETA env var. + +.. option:: -f HOSTFILE, --file HOSTFILE + + Read additional hosts to delete tags for from + specified file or from stdin if '-' (each host on + separate line). If no host or host file is specified + then, by default, read from stdin. If no tags/tagfile + nor hosts/hostfile are specified then tags are read + from stdin and are deleted from all hosts. Hostfile + format is the same as config hostfile format. + +.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR + + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if this + argument is set then specified directory is used, if + CDIST_INVENTORY_DIR env var is set then its value is + used, if HOME env var is set then ~/.cdist/inventory is + used, otherwise distribution inventory directory is + used. + +.. option:: -T TAGFILE, --tag-file TAGFILE + + Read additional tags from specified file or from stdin + if '-' (each tag on separate line). If no tag or tag + file is specified then, by default, read from stdin. + If no tags/tagfile nor hosts/hostfile are specified + then tags are read from stdin and are added to all + hosts. Tagfile format is the same as config hostfile format. + +.. option:: -t TAGLIST, --taglist TAGLIST + + Tag list to be deleted for specified host(s), comma + separated values + + +INVENTORY LIST +-------------- +List inventory database. + +.. option:: host + + host(s) to list + +.. option:: -a, --all + + list hosts that have all specified tags, if -t/--tag + is specified + +.. option:: -b, --beta + + Enable beta functionalities. Beta functionalities + include inventory command with all sub-commands and + all options; config sub-command options: -j/--jobs, + -t/--tag, -a/--all. + + Can also be enabled using CDIST_BETA env var. + +.. option:: -f HOSTFILE, --file HOSTFILE + + Read additional hosts to list from specified file or + from stdin if '-' (each host on separate line). If no + host or host file is specified then, by default, list + all. Hostfile format is the same as config hostfile format. + +.. option:: -H, --host-only + + Suppress tags listing + +.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR + + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if this + argument is set then specified directory is used, if + CDIST_INVENTORY_DIR env var is set then its value is + used, if HOME env var is set then ~/.cdist/inventory is + used, otherwise distribution inventory directory is + used. + +.. option:: -t, --tag + + host is specified by tag, not hostname/address; list + all hosts that contain any of specified tags + + SHELL ----- This command allows you to spawn a shell that enables access @@ -186,14 +474,21 @@ usage. Its primary use is for debugging type parameters. Select shell to use, defaults to current shell. Used shell should be POSIX compatible shell. + FILES ----- ~/.cdist Your personal cdist config directory. If exists it will be automatically used. +~/.cdist/inventory + The home inventory directory. If ~/.cdist exists it will be used as + default inventory directory. cdist/conf The distribution configuration directory. It contains official types and explorers. This path is relative to cdist installation directory. +cdist/inventory + The distribution inventory directory. + This path is relative to cdist installation directory. NOTES ----- @@ -243,6 +538,43 @@ EXAMPLES # Install ikq05.ethz.ch with debug enabled % cdist install -vvv ikq05.ethz.ch + # List inventory content + % cdist inventory list -b + + # List inventory for specified host localhost + % cdist inventory list -b localhost + + # List inventory for specified tag loadbalancer + % cdist inventory list -b -t loadbalancer + + # Add hosts to inventory + % cdist inventory add-host -b web1 web2 web3 + + # Delete hosts from file old-hosts from inventory + % cdist inventory del-host -b -f old-hosts + + # Add tags to specifed hosts + % cdist inventory add-tag -b -t europe,croatia,web,static web1 web2 + + # Add tag to all hosts in inventory + % cdist inventory add-tag -b -t vm + + # Delete all tags from specified host + % cdist inventory del-tag -b -a localhost + + # Delete tags read from stdin from hosts specified by file hosts + % cdist inventory del-tag -b -T - -f hosts + + # Configure hosts from inventory with any of specified tags + % cdist config -b -t web dynamic + + # Configure hosts from inventory with all specified tags + % cdist config -b -t -a web dynamic + + # Configure all hosts from inventory db + $ cdist config -b -A + + ENVIRONMENT ----------- TMPDIR, TEMP, TMP @@ -272,6 +604,9 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). +CDIST_INVENTORY_DIR + Use this directory as inventory directory. + CDIST_BETA Enable beta functionality. diff --git a/scripts/cdist b/scripts/cdist index 81220ca3..d9dce35f 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -33,6 +33,7 @@ def commandline(): import cdist.config import cdist.install import cdist.shell + import cdist.inventory import shutil import os