From e2a1519332e176e3a3c2d48e5a17b373070f03b5 Mon Sep 17 00:00:00 2001
From: Darko Poljak <foss@ungleich.com>
Date: Thu, 20 Jul 2017 22:04:44 +0200
Subject: [PATCH] Merge inventory from beta branch.
---
.gitignore | 3 +
cdist/argparse.py | 166 +++++-
cdist/conf/type/__install_stage/man.rst | 6 +-
cdist/config.py | 49 +-
cdist/core/code.py | 3 +
cdist/core/explorer.py | 1 +
cdist/core/manifest.py | 2 +
cdist/exec/local.py | 5 +
cdist/inventory.py | 390 ++++++++++++++
cdist/shell.py | 3 +
cdist/test/__init__.py | 1 +
cdist/test/code/__init__.py | 5 +
.../type/__dump_environment/gencode-local | 1 +
cdist/test/config/__init__.py | 3 +
cdist/test/emulator/__init__.py | 7 +
cdist/test/explorer/__init__.py | 1 +
cdist/test/inventory/__init__.py | 476 ++++++++++++++++++
cdist/test/manifest/__init__.py | 5 +
.../fixtures/conf/manifest/dump_environment | 1 +
.../conf/type/__dump_environment/manifest | 1 +
completions/bash/cdist-completion.bash | 59 ++-
completions/zsh/_cdist | 41 +-
docs/changelog | 4 +
docs/src/cdist-inventory.rst | 211 ++++++++
docs/src/cdist-reference.rst.sh | 10 +
docs/src/index.rst | 1 +
docs/src/man1/cdist.rst | 349 ++++++++++++-
scripts/cdist | 1 +
28 files changed, 1769 insertions(+), 36 deletions(-)
create mode 100644 cdist/inventory.py
create mode 100644 cdist/test/inventory/__init__.py
create mode 100644 docs/src/cdist-inventory.rst
diff --git a/.gitignore b/.gitignore
index 4258c2eb..55374d6d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,9 @@ docs/src/cdist-reference.rst
# Ignore cdist cache for version control
/cache/
+# Ignore inventory basedir
+cdist/inventory/
+
# Python: cache, distutils, distribution in general
__pycache__/
*.pyc
diff --git a/cdist/argparse.py b/cdist/argparse.py
index 8208d1f0..16b9d054 100644
--- a/cdist/argparse.py
+++ b/cdist/argparse.py
@@ -7,10 +7,10 @@ import collections
# set of beta sub-commands
-BETA_COMMANDS = set(('install', ))
+BETA_COMMANDS = set(('install', 'inventory', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
- 'config': set(('jobs', )),
+ 'config': set(('jobs', 'tag', 'all_tagged_hosts', )),
}
EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
# Parser others can reuse
@@ -121,6 +121,17 @@ def get_parsers():
'banner', parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
+ parser['inventory_common'] = argparse.ArgumentParser(add_help=False)
+ parser['inventory_common'].add_argument(
+ '-I', '--inventory',
+ help=('Use specified custom inventory directory. '
+ 'Inventory directory is set up by the following rules: '
+ 'if this argument is set then specified directory is used, '
+ 'if CDIST_INVENTORY_DIR env var is set then its value is '
+ 'used, if HOME env var is set then ~/.cdist/inventory is '
+ 'used, otherwise distribution inventory directory is used.'),
+ dest="inventory_dir", required=False)
+
# Config
parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
@@ -156,6 +167,10 @@ def get_parsers():
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
+ parser['config_main'].add_argument(
+ '-r', '--remote-out-dir',
+ help='Directory to save cdist output in on the target host',
+ dest="remote_out_path")
parser['config_main'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
@@ -170,6 +185,15 @@ def get_parsers():
# Config
parser['config_args'] = argparse.ArgumentParser(add_help=False)
+ parser['config_args'].add_argument(
+ '-A', '--all-tagged',
+ help=('use all hosts present in tags db'),
+ action="store_true", dest="all_tagged_hosts", default=False)
+ parser['config_args'].add_argument(
+ '-a', '--all',
+ help=('list hosts that have all specified tags, '
+ 'if -t/--tag is specified'),
+ action="store_true", dest="has_all_tags", default=False)
parser['config_args'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config_args'].add_argument(
@@ -183,17 +207,19 @@ def get_parsers():
'-p', '--parallel',
help='operate on multiple hosts in parallel',
action='store_true', dest='parallel')
- parser['config_args'].add_argument(
- '-r', '--remote-out-dir',
- help='Directory to save cdist output in on the target host',
- dest="remote_out_path")
parser['config_args'].add_argument(
'-s', '--sequential',
help='operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
+ parser['config_args'].add_argument(
+ '-t', '--tag',
+ help=('host is specified by tag, not hostname/address; '
+ 'list all hosts that contain any of specified tags'),
+ dest='tag', required=False, action="store_true", default=False)
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'],
parser['config_main'],
+ parser['inventory_common'],
parser['config_args']])
parser['config'].set_defaults(func=cdist.config.Config.commandline)
@@ -202,6 +228,134 @@ def get_parsers():
parents=[parser['config']])
parser['install'].set_defaults(func=cdist.install.Install.commandline)
+ # Inventory
+ parser['inventory'] = parser['sub'].add_parser(
+ 'inventory', parents=[parser['loglevel'], parser['beta'],
+ parser['inventory_common']])
+ parser['invsub'] = parser['inventory'].add_subparsers(
+ title="Inventory commands", dest="subcommand")
+
+ parser['add-host'] = parser['invsub'].add_parser(
+ 'add-host', parents=[parser['loglevel'], parser['beta'],
+ parser['inventory_common']])
+ parser['add-host'].add_argument(
+ 'host', nargs='*', help='host(s) to add')
+ parser['add-host'].add_argument(
+ '-f', '--file',
+ help=('Read additional hosts to add from specified file '
+ 'or from stdin if \'-\' (each host on separate line). '
+ 'If no host or host file is specified then, by default, '
+ 'read from stdin.'),
+ dest='hostfile', required=False)
+
+ parser['add-tag'] = parser['invsub'].add_parser(
+ 'add-tag', parents=[parser['loglevel'], parser['beta'],
+ parser['inventory_common']])
+ parser['add-tag'].add_argument(
+ 'host', nargs='*',
+ help='list of host(s) for which tags are added')
+ parser['add-tag'].add_argument(
+ '-f', '--file',
+ help=('Read additional hosts to add tags from specified file '
+ 'or from stdin if \'-\' (each host on separate line). '
+ 'If no host or host file is specified then, by default, '
+ 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+ ' are specified then tags are read from stdin and are'
+ ' added to all hosts.'),
+ dest='hostfile', required=False)
+ parser['add-tag'].add_argument(
+ '-T', '--tag-file',
+ help=('Read additional tags to add from specified file '
+ 'or from stdin if \'-\' (each tag on separate line). '
+ 'If no tag or tag file is specified then, by default, '
+ 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+ ' are specified then tags are read from stdin and are'
+ ' added to all hosts.'),
+ dest='tagfile', required=False)
+ parser['add-tag'].add_argument(
+ '-t', '--taglist',
+ help=("Tag list to be added for specified host(s), comma separated"
+ " values"),
+ dest="taglist", required=False)
+
+ parser['del-host'] = parser['invsub'].add_parser(
+ 'del-host', parents=[parser['loglevel'], parser['beta'],
+ parser['inventory_common']])
+ parser['del-host'].add_argument(
+ 'host', nargs='*', help='host(s) to delete')
+ parser['del-host'].add_argument(
+ '-a', '--all', help=('Delete all hosts'),
+ dest='all', required=False, action="store_true", default=False)
+ parser['del-host'].add_argument(
+ '-f', '--file',
+ help=('Read additional hosts to delete from specified file '
+ 'or from stdin if \'-\' (each host on separate line). '
+ 'If no host or host file is specified then, by default, '
+ 'read from stdin.'),
+ dest='hostfile', required=False)
+
+ parser['del-tag'] = parser['invsub'].add_parser(
+ 'del-tag', parents=[parser['loglevel'], parser['beta'],
+ parser['inventory_common']])
+ parser['del-tag'].add_argument(
+ 'host', nargs='*',
+ help='list of host(s) for which tags are deleted')
+ parser['del-tag'].add_argument(
+ '-a', '--all',
+ help=('Delete all tags for specified host(s)'),
+ dest='all', required=False, action="store_true", default=False)
+ parser['del-tag'].add_argument(
+ '-f', '--file',
+ help=('Read additional hosts to delete tags for from specified '
+ 'file or from stdin if \'-\' (each host on separate line). '
+ 'If no host or host file is specified then, by default, '
+ 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+ ' are specified then tags are read from stdin and are'
+ ' deleted from all hosts.'),
+ dest='hostfile', required=False)
+ parser['del-tag'].add_argument(
+ '-T', '--tag-file',
+ help=('Read additional tags from specified file '
+ 'or from stdin if \'-\' (each tag on separate line). '
+ 'If no tag or tag file is specified then, by default, '
+ 'read from stdin. If no tags/tagfile nor'
+ ' hosts/hostfile are specified then tags are read from'
+ ' stdin and are added to all hosts.'),
+ dest='tagfile', required=False)
+ parser['del-tag'].add_argument(
+ '-t', '--taglist',
+ help=("Tag list to be deleted for specified host(s), "
+ "comma separated values"),
+ dest="taglist", required=False)
+
+ parser['list'] = parser['invsub'].add_parser(
+ 'list', parents=[parser['loglevel'], parser['beta'],
+ parser['inventory_common']])
+ parser['list'].add_argument(
+ 'host', nargs='*', help='host(s) to list')
+ parser['list'].add_argument(
+ '-a', '--all',
+ help=('list hosts that have all specified tags, '
+ 'if -t/--tag is specified'),
+ action="store_true", dest="has_all_tags", default=False)
+ parser['list'].add_argument(
+ '-f', '--file',
+ help=('Read additional hosts to list from specified file '
+ 'or from stdin if \'-\' (each host on separate line). '
+ 'If no host or host file is specified then, by default, '
+ 'list all.'), dest='hostfile', required=False)
+ parser['list'].add_argument(
+ '-H', '--host-only', help=('Suppress tags listing'),
+ action="store_true", dest="list_only_host", default=False)
+ parser['list'].add_argument(
+ '-t', '--tag',
+ help=('host is specified by tag, not hostname/address; '
+ 'list all hosts that contain any of specified tags'),
+ action="store_true", default=False)
+
+ parser['inventory'].set_defaults(
+ func=cdist.inventory.Inventory.commandline)
+
# Shell
parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']])
diff --git a/cdist/conf/type/__install_stage/man.rst b/cdist/conf/type/__install_stage/man.rst
index 6c68c543..e33e1e90 100644
--- a/cdist/conf/type/__install_stage/man.rst
+++ b/cdist/conf/type/__install_stage/man.rst
@@ -17,9 +17,9 @@ REQUIRED PARAMETERS
uri
The uri from which to fetch the tarball.
Can be anything understood by curl, e.g:
- | http://path/to/stage.tgz
- | tftp:///path/to/stage.tgz
- | file:///local/path/stage.tgz
+ | http://path/to/stage.tgz
+ | tftp:///path/to/stage.tgz
+ | file:///local/path/stage.tgz
OPTIONAL PARAMETERS
diff --git a/cdist/config.py b/cdist/config.py
index 450fde29..2c9721f5 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -38,6 +38,9 @@ import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
+
+from cdist import inventory
+
import cdist.util.ipaddr as ipaddr
from cdist import core
@@ -134,11 +137,42 @@ class Config(object):
base_root_path = cls.create_base_root_path(args.out_path)
hostcnt = 0
- for host in itertools.chain(cls.hosts(args.host),
- cls.hosts(args.hostfile)):
+
+ if args.tag or args.all_tagged_hosts:
+ inventory.determine_default_inventory_dir(args)
+ if args.all_tagged_hosts:
+ inv_list = inventory.InventoryList(
+ hosts=None, istag=True, hostfile=None,
+ db_basedir=args.inventory_dir)
+ else:
+ inv_list = inventory.InventoryList(
+ hosts=args.host, istag=True, hostfile=args.hostfile,
+ db_basedir=args.inventory_dir,
+ has_all_tags=args.has_all_tags)
+ it = inv_list.entries()
+ else:
+ it = itertools.chain(cls.hosts(args.host),
+ cls.hosts(args.hostfile))
+ for entry in it:
+ if isinstance(entry, tuple):
+ # if configuring by specified tags
+ host = entry[0]
+ host_tags = entry[1]
+ else:
+ # if configuring by host then check inventory for tags
+ host = entry
+ inventory.determine_default_inventory_dir(args)
+ inv_list = inventory.InventoryList(
+ hosts=(host,), db_basedir=args.inventory_dir)
+ inv = tuple(inv_list.entries())
+ if inv:
+ # host is present in inventory and has tags
+ host_tags = inv[0][1]
+ else:
+ # host is not present in inventory or has no tags
+ host_tags = None
host_base_path, hostdir = cls.create_host_base_dirs(
host, base_root_path)
-
log.debug("Base root path for target host \"{}\" is \"{}\"".format(
host, host_base_path))
@@ -147,11 +181,12 @@ class Config(object):
log.trace("Creating child process for %s", host)
process[host] = multiprocessing.Process(
target=cls.onehost,
- args=(host, host_base_path, hostdir, args, True))
+ args=(host, host_tags, host_base_path, hostdir, args,
+ True))
process[host].start()
else:
try:
- cls.onehost(host, host_base_path, hostdir,
+ cls.onehost(host, host_tags, host_base_path, hostdir,
args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
@@ -199,7 +234,8 @@ class Config(object):
return (remote_exec, remote_copy, )
@classmethod
- def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
+ def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
+ parallel):
"""Configure ONE system"""
log = logging.getLogger(host)
@@ -216,6 +252,7 @@ class Config(object):
local = cdist.exec.local.Local(
target_host=target_host,
+ target_host_tags=host_tags,
base_root_path=host_base_path,
host_dir_name=host_dir_name,
initial_manifest=args.manifest,
diff --git a/cdist/core/code.py b/cdist/core/code.py
index 1b5fe1d6..173d192d 100644
--- a/cdist/core/code.py
+++ b/cdist/core/code.py
@@ -56,6 +56,7 @@ gencode-local
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type: full qualified path to the type's dir
__files: full qualified path to the files dir
+ __target_host_tags: comma spearated list of host tags
returns: string containing the generated code or None
@@ -74,6 +75,7 @@ gencode-remote
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type: full qualified path to the type's dir
__files: full qualified path to the files dir
+ __target_host_tags: comma spearated list of host tags
returns: string containing the generated code or None
@@ -106,6 +108,7 @@ class Code(object):
'__target_fqdn': self.target_host[2],
'__global': self.local.base_path,
'__files': self.local.files_path,
+ '__target_host_tags': self.local.target_host_tags,
}
def _run_gencode(self, cdist_object, which):
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index d604c015..38f2a921 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -77,6 +77,7 @@ class Explorer(object):
'__target_hostname': self.target_host[1],
'__target_fqdn': self.target_host[2],
'__explorer': self.remote.global_explorer_path,
+ '__target_host_tags': self.local.target_host_tags,
}
self._type_explorers_transferred = []
self.jobs = jobs
diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py
index d8570097..6f941550 100644
--- a/cdist/core/manifest.py
+++ b/cdist/core/manifest.py
@@ -42,6 +42,7 @@ common:
types are defined for use in type emulator
== local.type_path
__files: full qualified path to the files dir
+ __target_host_tags: comma spearated list of host tags
initial manifest is:
script: full qualified path to the initial manifest
@@ -109,6 +110,7 @@ class Manifest(object):
'__target_hostname': self.target_host[1],
'__target_fqdn': self.target_host[2],
'__files': self.local.files_path,
+ '__target_host_tags': self.local.target_host_tags,
}
if self.log.getEffectiveLevel() == logging.DEBUG:
diff --git a/cdist/exec/local.py b/cdist/exec/local.py
index 6c285204..23ad4ce9 100644
--- a/cdist/exec/local.py
+++ b/cdist/exec/local.py
@@ -49,6 +49,7 @@ class Local(object):
"""
def __init__(self,
target_host,
+ target_host_tags,
base_root_path,
host_dir_name,
exec_path=sys.argv[0],
@@ -58,6 +59,10 @@ class Local(object):
quiet_mode=False):
self.target_host = target_host
+ if target_host_tags is None:
+ self.target_host_tags = ""
+ else:
+ self.target_host_tags = ",".join(target_host_tags)
self.hostdir = host_dir_name
self.base_path = os.path.join(base_root_path, "data")
diff --git a/cdist/inventory.py b/cdist/inventory.py
new file mode 100644
index 00000000..ccb4428f
--- /dev/null
+++ b/cdist/inventory.py
@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import cdist
+import logging
+import os
+import os.path
+import itertools
+import sys
+from cdist.hostsource import hostfile_process_line
+
+DIST_INVENTORY_DB_NAME = "inventory"
+
+dist_inventory_db = os.path.abspath(os.path.join(
+ os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME))
+
+
+def determine_default_inventory_dir(args):
+ # The order of inventory dir setting by decreasing priority
+ # 1. inventory_dir argument
+ # 2. CDIST_INVENTORY_DIR env var if set
+ # 3. ~/.cdist/inventory if HOME env var is set
+ # 4. distribution inventory directory
+ if not args.inventory_dir:
+ if 'CDIST_INVENTORY_DIR' in os.environ:
+ args.inventory_dir = os.environ['CDIST_INVENTORY_DIR']
+ else:
+ home = cdist.home_dir()
+ if home:
+ args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
+ else:
+ args.inventory_dir = dist_inventory_db
+
+
+def contains_all(big, little):
+ """Return True if big contains all elements from little,
+ False otherwise.
+ """
+ return set(little).issubset(set(big))
+
+
+def contains_any(big, little):
+ """Return True if big contains any element from little,
+ False otherwise.
+ """
+ for x in little:
+ if x in big:
+ return True
+ return False
+
+
+def check_always_true(x, y):
+ return True
+
+
+def rstrip_nl(s):
+ '''str.rstrip "\n" from s'''
+ return str.rstrip(s, "\n")
+
+
+class Inventory(object):
+ """Inventory main class"""
+
+ def __init__(self, db_basedir=dist_inventory_db):
+ self.db_basedir = db_basedir
+ self.log = logging.getLogger("inventory")
+ self.init_db()
+
+ def init_db(self):
+ self.log.debug("Init db: {}".format(self.db_basedir))
+ if not os.path.exists(self.db_basedir):
+ os.makedirs(self.db_basedir, exist_ok=True)
+ elif not os.path.isdir(self.db_basedir):
+ raise cdist.Error(("Invalid inventory db basedir \'{}\',"
+ " must be a directory").format(self.db_basedir))
+
+ @staticmethod
+ def strlist_to_list(slist):
+ if slist:
+ result = [x for x in slist.split(',') if x]
+ else:
+ result = []
+ return result
+
+ def _input_values(self, source):
+ """Yield input values from source.
+ Source can be a sequence or filename (stdin if '-').
+ In case of filename each line represents one input value.
+ """
+ if isinstance(source, str):
+ import fileinput
+ try:
+ with fileinput.FileInput(files=(source)) as f:
+ for x in f:
+ result = hostfile_process_line(x, strip_func=rstrip_nl)
+ if result:
+ yield result
+ except (IOError, OSError) as e:
+ raise cdist.Error("Error reading from \'{}\'".format(
+ source))
+ else:
+ if source:
+ for x in source:
+ if x:
+ yield x
+
+ def _host_path(self, host):
+ hostpath = os.path.join(self.db_basedir, host)
+ return hostpath
+
+ def _all_hosts(self):
+ return os.listdir(self.db_basedir)
+
+ def _check_host(self, hostpath):
+ if not os.path.exists(hostpath):
+ return False
+ else:
+ if not os.path.isfile(hostpath):
+ raise cdist.Error(("Host path \'{}\' exists, but is not"
+ " a valid file").format(hostpath))
+ return True
+
+ def _read_host_tags(self, hostpath):
+ result = set()
+ with open(hostpath, "rt") as f:
+ for tag in f:
+ tag = tag.rstrip("\n")
+ if tag:
+ result.add(tag)
+ return result
+
+ def _get_host_tags(self, host):
+ hostpath = self._host_path(host)
+ if self._check_host(hostpath):
+ return self._read_host_tags(hostpath)
+ else:
+ return None
+
+ def _write_host_tags(self, host, tags):
+ hostpath = self._host_path(host)
+ if self._check_host(hostpath):
+ with open(hostpath, "wt") as f:
+ for tag in tags:
+ f.write("{}\n".format(tag))
+ return True
+ else:
+ return False
+
+ @classmethod
+ def commandline(cls, args):
+ """Manipulate inventory db"""
+ log = logging.getLogger("cdist")
+ if 'taglist' in args:
+ args.taglist = cls.strlist_to_list(args.taglist)
+ determine_default_inventory_dir(args)
+
+ log.info("Using inventory: {}".format(args.inventory_dir))
+ log.debug("Inventory args: {}".format(vars(args)))
+ log.debug("Inventory command: {}".format(args.subcommand))
+
+ if args.subcommand == "list":
+ c = InventoryList(hosts=args.host, istag=args.tag,
+ hostfile=args.hostfile,
+ db_basedir=args.inventory_dir,
+ list_only_host=args.list_only_host,
+ has_all_tags=args.has_all_tags)
+ elif args.subcommand == "add-host":
+ c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
+ db_basedir=args.inventory_dir)
+ elif args.subcommand == "del-host":
+ c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
+ all=args.all, db_basedir=args.inventory_dir,
+ action="del")
+ elif args.subcommand == "add-tag":
+ c = InventoryTag(hosts=args.host, tags=args.taglist,
+ hostfile=args.hostfile, tagfile=args.tagfile,
+ db_basedir=args.inventory_dir)
+ elif args.subcommand == "del-tag":
+ c = InventoryTag(hosts=args.host, tags=args.taglist,
+ hostfile=args.hostfile, tagfile=args.tagfile,
+ all=args.all, db_basedir=args.inventory_dir,
+ action="del")
+ else:
+ raise cdist.Error("Unknown inventory command \'{}\'".format(
+ args.subcommand))
+ c.run()
+
+
+class InventoryList(Inventory):
+ def __init__(self, hosts=None, istag=False, hostfile=None,
+ list_only_host=False, has_all_tags=False,
+ db_basedir=dist_inventory_db):
+ super().__init__(db_basedir)
+ self.hosts = hosts
+ self.istag = istag
+ self.hostfile = hostfile
+ self.list_only_host = list_only_host
+ self.has_all_tags = has_all_tags
+
+ def _print(self, host, tags):
+ if self.list_only_host:
+ print("{}".format(host))
+ else:
+ print("{} {}".format(host, ",".join(sorted(tags))))
+
+ def _do_list(self, it_tags, it_hosts, check_func):
+ if (it_tags is not None):
+ param_tags = set(it_tags)
+ self.log.debug("param_tags: {}".format(param_tags))
+ else:
+ param_tags = set()
+ for host in it_hosts:
+ self.log.debug("host: {}".format(host))
+ tags = self._get_host_tags(host)
+ if tags is None:
+ self.log.info("Host \'{}\' not found, skipped".format(host))
+ continue
+ self.log.debug("tags: {}".format(tags))
+ if check_func(tags, param_tags):
+ yield host, tags
+
+ def entries(self):
+ if not self.hosts and not self.hostfile:
+ self.log.info("Listing all hosts")
+ it_hosts = self._all_hosts()
+ it_tags = None
+ check_func = check_always_true
+ else:
+ it = itertools.chain(self._input_values(self.hosts),
+ self._input_values(self.hostfile))
+ if self.istag:
+ self.log.info("Listing by tag(s)")
+ it_hosts = self._all_hosts()
+ it_tags = it
+ if self.has_all_tags:
+ check_func = contains_all
+ else:
+ check_func = contains_any
+ else:
+ self.log.info("Listing by host(s)")
+ it_hosts = it
+ it_tags = None
+ check_func = check_always_true
+ for host, tags in self._do_list(it_tags, it_hosts, check_func):
+ yield host, tags
+
+ def host_entries(self):
+ for host, tags in self.entries():
+ yield host
+
+ def run(self):
+ for host, tags in self.entries():
+ self._print(host, tags)
+
+
+class InventoryHost(Inventory):
+ def __init__(self, hosts=None, hostfile=None,
+ db_basedir=dist_inventory_db, all=False, action="add"):
+ super().__init__(db_basedir)
+ self.actions = ("add", "del")
+ if action not in self.actions:
+ raise cdist.Error("Invalid action \'{}\', valid actions are:"
+ " {}\n".format(action, self.actions.keys()))
+ self.action = action
+ self.hosts = hosts
+ self.hostfile = hostfile
+ self.all = all
+
+ if not self.hosts and not self.hostfile:
+ self.hostfile = "-"
+
+ def _new_hostpath(self, hostpath):
+ # create empty file
+ with open(hostpath, "w"):
+ pass
+
+ def _action(self, host):
+ if self.action == "add":
+ self.log.info("Adding host \'{}\'".format(host))
+ elif self.action == "del":
+ self.log.info("Deleting host \'{}\'".format(host))
+ hostpath = self._host_path(host)
+ self.log.debug("hostpath: {}".format(hostpath))
+ if self.action == "add" and not os.path.exists(hostpath):
+ self._new_hostpath(hostpath)
+ else:
+ if not os.path.isfile(hostpath):
+ raise cdist.Error(("Host path \'{}\' is"
+ " not a valid file").format(hostpath))
+ if self.action == "del":
+ os.remove(hostpath)
+
+ def run(self):
+ if self.action == "del" and self.all:
+ self.log.debug("Doing for all hosts")
+ it = self._all_hosts()
+ else:
+ self.log.debug("Doing for specified hosts")
+ it = itertools.chain(self._input_values(self.hosts),
+ self._input_values(self.hostfile))
+ for host in it:
+ self._action(host)
+
+
+class InventoryTag(Inventory):
+ def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None,
+ db_basedir=dist_inventory_db, all=False, action="add"):
+ super().__init__(db_basedir)
+ self.actions = ("add", "del")
+ if action not in self.actions:
+ raise cdist.Error("Invalid action \'{}\', valid actions are:"
+ " {}\n".format(action, self.actions.keys()))
+ self.action = action
+ self.hosts = hosts
+ self.tags = tags
+ self.hostfile = hostfile
+ self.tagfile = tagfile
+ self.all = all
+
+ if not self.hosts and not self.hostfile:
+ self.allhosts = True
+ else:
+ self.allhosts = False
+ if not self.tags and not self.tagfile:
+ self.tagfile = "-"
+
+ if self.hostfile == "-" and self.tagfile == "-":
+ raise cdist.Error("Cannot read both, hosts and tags, from stdin")
+
+ def _read_input_tags(self):
+ self.input_tags = set()
+ for tag in itertools.chain(self._input_values(self.tags),
+ self._input_values(self.tagfile)):
+ self.input_tags.add(tag)
+
+ def _action(self, host):
+ host_tags = self._get_host_tags(host)
+ if host_tags is None:
+ print("Host \'{}\' does not exist, skipping".format(host),
+ file=sys.stderr)
+ return
+ self.log.debug("existing host_tags: {}".format(host_tags))
+ if self.action == "del" and self.all:
+ host_tags = set()
+ else:
+ for tag in self.input_tags:
+ if self.action == "add":
+ self.log.info("Adding tag \'{}\' for host \'{}\'".format(
+ tag, host))
+ host_tags.add(tag)
+ elif self.action == "del":
+ self.log.info("Deleting tag \'{}\' for host \'{}\'".format(
+ tag, host))
+ if tag in host_tags:
+ host_tags.remove(tag)
+ self.log.debug("new host tags: {}".format(host_tags))
+ if not self._write_host_tags(host, host_tags):
+ self.log.info("{} does not exist, skipped".format(host))
+
+ def run(self):
+ if self.allhosts:
+ self.log.debug("Doing for all hosts")
+ it = self._all_hosts()
+ else:
+ self.log.debug("Doing for specified hosts")
+ it = itertools.chain(self._input_values(self.hosts),
+ self._input_values(self.hostfile))
+ if not(self.action == "del" and self.all):
+ self._read_input_tags()
+ for host in it:
+ self._action(host)
diff --git a/cdist/shell.py b/cdist/shell.py
index 662f8f7d..44cdd49d 100644
--- a/cdist/shell.py
+++ b/cdist/shell.py
@@ -44,6 +44,7 @@ class Shell(object):
"cdist-shell-no-target-host",
"cdist-shell-no-target-host",
)
+ self.target_host_tags = ""
host_dir_name = cdist.str_hash(self.target_host[0])
base_root_path = tempfile.mkdtemp()
@@ -51,6 +52,7 @@ class Shell(object):
self.local = cdist.exec.local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=host_dir_name)
@@ -77,6 +79,7 @@ class Shell(object):
'__manifest': self.local.manifest_path,
'__explorer': self.local.global_explorer_path,
'__files': self.local.files_path,
+ '__target_host_tags': self.local.target_host_tags,
}
self.env.update(additional_env)
diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py
index 83b0c618..faa3686a 100644
--- a/cdist/test/__init__.py
+++ b/cdist/test/__init__.py
@@ -42,6 +42,7 @@ class CdistTestCase(unittest.TestCase):
'cdisttesthost',
'cdisttesthost',
)
+ target_host_tags = "tag1,tag2,tag3"
def mkdtemp(self, **kwargs):
return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)
diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py
index 83c93f8b..50da2b8a 100644
--- a/cdist/test/code/__init__.py
+++ b/cdist/test/code/__init__.py
@@ -46,6 +46,7 @@ class CodeTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=cdist.test.cdist_exec_path,
@@ -97,6 +98,8 @@ class CodeTestCase(test.CdistTestCase):
self.cdist_object.object_id)
self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
self.assertEqual(output_dict['__files'], self.local.files_path)
+ self.assertEqual(output_dict['__target_host_tags'],
+ self.local.target_host_tags)
def test_run_gencode_remote_environment(self):
output_string = self.code.run_gencode_remote(self.cdist_object)
@@ -120,6 +123,8 @@ class CodeTestCase(test.CdistTestCase):
self.cdist_object.object_id)
self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
self.assertEqual(output_dict['__files'], self.local.files_path)
+ self.assertEqual(output_dict['__target_host_tags'],
+ self.local.target_host_tags)
def test_transfer_code_remote(self):
self.cdist_object.code_remote = self.code.run_gencode_remote(
diff --git a/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local b/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local
index 7fa70342..56744a27 100755
--- a/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local
+++ b/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local
@@ -9,3 +9,4 @@ echo "echo __object: $__object"
echo "echo __object_id: $__object_id"
echo "echo __object_name: $__object_name"
echo "echo __files: $__files"
+echo "echo __target_host_tags: $__target_host_tags"
diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py
index af1aa38f..5ff98269 100644
--- a/cdist/test/config/__init__.py
+++ b/cdist/test/config/__init__.py
@@ -60,6 +60,7 @@ class ConfigRunTestCase(test.CdistTestCase):
os.makedirs(self.host_base_path)
self.local = cdist.exec.local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir)
@@ -164,6 +165,7 @@ class ConfigRunTestCase(test.CdistTestCase):
"""Test if the dryrun option is working like expected"""
drylocal = cdist.exec.local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
# exec_path can not derivated from sys.argv in case of unittest
@@ -181,6 +183,7 @@ class ConfigRunTestCase(test.CdistTestCase):
"""Test to show dependency resolver warning message."""
local = cdist.exec.local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=os.path.abspath(os.path.join(
diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py
index 51de3180..664ab20b 100644
--- a/cdist/test/emulator/__init__.py
+++ b/cdist/test/emulator/__init__.py
@@ -53,6 +53,7 @@ class EmulatorTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@@ -156,6 +157,7 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@@ -246,6 +248,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@@ -279,6 +282,7 @@ class OverrideTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@@ -322,6 +326,7 @@ class ArgumentsTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@@ -445,6 +450,7 @@ class StdinTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@@ -511,6 +517,7 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py
index fc66020d..928b4e0d 100644
--- a/cdist/test/explorer/__init__.py
+++ b/cdist/test/explorer/__init__.py
@@ -50,6 +50,7 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=base_root_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py
new file mode 100644
index 00000000..4c0dd936
--- /dev/null
+++ b/cdist/test/inventory/__init__.py
@@ -0,0 +1,476 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import shutil
+import cdist
+import os.path as op
+import unittest
+import sys
+from cdist import test
+from cdist import inventory
+from io import StringIO
+
+my_dir = op.abspath(op.dirname(__file__))
+fixtures = op.join(my_dir, 'fixtures')
+inventory_dir = op.join(fixtures, "inventory")
+
+
+class InventoryTestCase(test.CdistTestCase):
+
+ def _create_host_with_tags(self, host, tags):
+ os.makedirs(inventory_dir, exist_ok=True)
+ hostfile = op.join(inventory_dir, host)
+ with open(hostfile, "w") as f:
+ for x in tags:
+ f.write("{}\n".format(x))
+
+ def setUp(self):
+ self.maxDiff = None
+ self.db = {
+ "loadbalancer1": ["loadbalancer", "all", "europe", ],
+ "loadbalancer2": ["loadbalancer", "all", "europe", ],
+ "loadbalancer3": ["loadbalancer", "all", "africa", ],
+ "loadbalancer4": ["loadbalancer", "all", "africa", ],
+ "web1": ["web", "all", "static", ],
+ "web2": ["web", "all", "dynamic", ],
+ "web3": ["web", "all", "dynamic", ],
+ "shell1": ["shell", "all", "free", ],
+ "shell2": ["shell", "all", "free", ],
+ "shell3": ["shell", "all", "charge", ],
+ "shell4": ["shell", "all", "charge", ],
+ "monty": ["web", "python", "shell", ],
+ "python": ["web", "python", "shell", ],
+ }
+ for x in self.db:
+ self.db[x] = sorted(self.db[x])
+ for host in self.db:
+ self._create_host_with_tags(host, self.db[host])
+ self.sys_stdout = sys.stdout
+ out = StringIO()
+ sys.stdout = out
+
+ def _get_output(self):
+ sys.stdout.flush()
+ output = sys.stdout.getvalue().strip()
+ return output
+
+ def tearDown(self):
+ sys.stdout = self.sys_stdout
+ shutil.rmtree(inventory_dir)
+
+ def test_inventory_create_db(self):
+ dbdir = op.join(fixtures, "foo")
+ inv = inventory.Inventory(db_basedir=dbdir)
+ self.assertTrue(os.path.isdir(dbdir))
+ self.assertEqual(inv.db_basedir, dbdir)
+ shutil.rmtree(inv.db_basedir)
+
+ # InventoryList
+ def test_inventory_list_print(self):
+ invList = inventory.InventoryList(db_basedir=inventory_dir)
+ invList.run()
+ output = self._get_output()
+ self.assertTrue(' ' in output)
+
+ def test_inventory_list_print_host_only(self):
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ list_only_host=True)
+ invList.run()
+ output = self._get_output()
+ self.assertFalse(' ' in output)
+
+ def test_inventory_list_all(self):
+ invList = inventory.InventoryList(db_basedir=inventory_dir)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ self.assertEqual(db, self.db)
+
+ def test_inventory_list_by_host_hosts(self):
+ hosts = ("web1", "web2", "web3",)
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ expected_db = {host: sorted(self.db[host]) for host in hosts}
+ self.assertEqual(db, expected_db)
+
+ def test_inventory_list_by_host_hostfile(self):
+ hosts = ("web1", "web2", "web3",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hosts:
+ f.write("{}\n".format(x))
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hostfile=hostfile)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ expected_db = {host: sorted(self.db[host]) for host in hosts}
+ self.assertEqual(db, expected_db)
+ os.remove(hostfile)
+
+ def test_inventory_list_by_host_hosts_hostfile(self):
+ hosts = ("shell1", "shell4",)
+ hostsf = ("web1", "web2", "web3",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hostsf:
+ f.write("{}\n".format(x))
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts, hostfile=hostfile)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ import itertools
+ expected_db = {host: sorted(self.db[host]) for host in
+ itertools.chain(hostsf, hosts)}
+ self.assertEqual(db, expected_db)
+ os.remove(hostfile)
+
+ def _gen_expected_db_for_tags(self, tags):
+ db = {}
+ for host in self.db:
+ for tag in tags:
+ if tag in self.db[host]:
+ db[host] = self.db[host]
+ break
+ return db
+
+ def _gen_expected_db_for_has_all_tags(self, tags):
+ db = {}
+ for host in self.db:
+ if set(tags).issubset(set(self.db[host])):
+ db[host] = self.db[host]
+ return db
+
+ def test_inventory_list_by_tag_hosts(self):
+ tags = ("web", "shell",)
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ istag=True, hosts=tags)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ expected_db = self._gen_expected_db_for_tags(tags)
+ self.assertEqual(db, expected_db)
+
+ def test_inventory_list_by_tag_hostfile(self):
+ tags = ("web", "shell",)
+ tagfile = op.join(fixtures, "tags")
+ with open(tagfile, "w") as f:
+ for x in tags:
+ f.write("{}\n".format(x))
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ istag=True, hostfile=tagfile)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ expected_db = self._gen_expected_db_for_tags(tags)
+ self.assertEqual(db, expected_db)
+ os.remove(tagfile)
+
+ def test_inventory_list_by_tag_hosts_hostfile(self):
+ tags = ("web", "shell",)
+ tagsf = ("dynamic", "europe",)
+ tagfile = op.join(fixtures, "tags")
+ with open(tagfile, "w") as f:
+ for x in tagsf:
+ f.write("{}\n".format(x))
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ istag=True, hosts=tags,
+ hostfile=tagfile)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ import itertools
+ expected_db = self._gen_expected_db_for_tags(tags + tagsf)
+ self.assertEqual(db, expected_db)
+ os.remove(tagfile)
+
+ def test_inventory_list_by_tag_has_all_tags(self):
+ tags = ("web", "python", "shell",)
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ istag=True, hosts=tags,
+ has_all_tags=True)
+ entries = invList.entries()
+ db = {host: sorted(tags) for host, tags in entries}
+ expected_db = self._gen_expected_db_for_has_all_tags(tags)
+ self.assertEqual(db, expected_db)
+
+ # InventoryHost
+ def test_inventory_host_add_hosts(self):
+ hosts = ("spam", "eggs", "foo",)
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="add", hosts=hosts)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir)
+ expected_hosts = tuple(x for x in invList.host_entries() if x in hosts)
+ self.assertEqual(sorted(hosts), sorted(expected_hosts))
+
+ def test_inventory_host_add_hostfile(self):
+ hosts = ("spam-new", "eggs-new", "foo-new",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hosts:
+ f.write("{}\n".format(x))
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="add", hostfile=hostfile)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir)
+ expected_hosts = tuple(x for x in invList.host_entries() if x in hosts)
+ self.assertEqual(sorted(hosts), sorted(expected_hosts))
+ os.remove(hostfile)
+
+ def test_inventory_host_add_hosts_hostfile(self):
+ hosts = ("spam-spam", "eggs-spam", "foo-spam",)
+ hostf = ("spam-eggs-spam", "spam-foo-spam",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hostf:
+ f.write("{}\n".format(x))
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="add", hosts=hosts,
+ hostfile=hostfile)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts + hostf)
+ expected_hosts = tuple(invList.host_entries())
+ self.assertEqual(sorted(hosts + hostf), sorted(expected_hosts))
+ os.remove(hostfile)
+
+ def test_inventory_host_del_hosts(self):
+ hosts = ("web1", "shell1",)
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="del", hosts=hosts)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts)
+ expected_hosts = tuple(invList.host_entries())
+ self.assertTupleEqual(expected_hosts, ())
+
+ def test_inventory_host_del_hostfile(self):
+ hosts = ("loadbalancer3", "loadbalancer4",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hosts:
+ f.write("{}\n".format(x))
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="del", hostfile=hostfile)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts)
+ expected_hosts = tuple(invList.host_entries())
+ self.assertTupleEqual(expected_hosts, ())
+ os.remove(hostfile)
+
+ def test_inventory_host_del_hosts_hostfile(self):
+ hosts = ("loadbalancer1", "loadbalancer2",)
+ hostf = ("web2", "shell2",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hostf:
+ f.write("{}\n".format(x))
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="del", hosts=hosts,
+ hostfile=hostfile)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts + hostf)
+ expected_hosts = tuple(invList.host_entries())
+ self.assertTupleEqual(expected_hosts, ())
+ os.remove(hostfile)
+
+ @unittest.expectedFailure
+ def test_inventory_host_invalid_host(self):
+ try:
+ invalid_hostfile = op.join(inventory_dir, "invalid")
+ os.mkdir(invalid_hostfile)
+ hosts = ("invalid",)
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="del", hosts=hosts)
+ invHost.run()
+ except e:
+ os.rmdir(invalid_hostfile)
+ raise e
+
+ # InventoryTag
+ def test_inventory_tag_init(self):
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="add")
+ self.assertTrue(invTag.allhosts)
+ self.assertEqual(invTag.tagfile, "-")
+
+ def test_inventory_tag_stdin_multiple_hosts(self):
+ try:
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="add", tagfile="-",
+ hosts=("host1", "host2",))
+ except e:
+ self.fail()
+
+ def test_inventory_tag_stdin_hostfile(self):
+ try:
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="add", tagfile="-",
+ hostfile="hosts")
+ except e:
+ self.fail()
+
+ @unittest.expectedFailure
+ def test_inventory_tag_stdin_both(self):
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="add", tagfile="-",
+ hostfile="-")
+
+ def test_inventory_tag_add_for_all_hosts(self):
+ tags = ("spam-spam-spam", "spam-spam-eggs",)
+ tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",)
+ tagfile = op.join(fixtures, "tags")
+ with open(tagfile, "w") as f:
+ for x in tagsf:
+ f.write("{}\n".format(x))
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="add", tags=tags,
+ tagfile=tagfile)
+ invTag.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir)
+ failed = False
+ for host, taglist in invList.entries():
+ for x in tagsf + tags:
+ if x not in taglist:
+ failed = True
+ break
+ if failed:
+ break
+ os.remove(tagfile)
+ if failed:
+ self.fail()
+
+ def test_inventory_tag_add(self):
+ tags = ("spam-spam-spam", "spam-spam-eggs",)
+ tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",)
+ hosts = ("loadbalancer1", "loadbalancer2", "shell2",)
+ hostsf = ("web2", "web3",)
+ tagfile = op.join(fixtures, "tags")
+ with open(tagfile, "w") as f:
+ for x in tagsf:
+ f.write("{}\n".format(x))
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hostsf:
+ f.write("{}\n".format(x))
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="add", tags=tags,
+ tagfile=tagfile, hosts=hosts,
+ hostfile=hostfile)
+ invTag.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts + hostsf)
+ failed = False
+ for host, taglist in invList.entries():
+ if host not in hosts + hostsf:
+ failed = True
+ break
+ for x in tagsf + tags:
+ if x not in taglist:
+ failed = True
+ break
+ if failed:
+ break
+ os.remove(tagfile)
+ os.remove(hostfile)
+ if failed:
+ self.fail()
+
+ def test_inventory_tag_del_for_all_hosts(self):
+ tags = ("all",)
+ tagsf = ("charge",)
+ tagfile = op.join(fixtures, "tags")
+ with open(tagfile, "w") as f:
+ for x in tagsf:
+ f.write("{}\n".format(x))
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="del", tags=tags,
+ tagfile=tagfile)
+ invTag.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir)
+ failed = False
+ for host, taglist in invList.entries():
+ for x in tagsf + tags:
+ if x in taglist:
+ failed = True
+ break
+ if failed:
+ break
+ os.remove(tagfile)
+ if failed:
+ self.fail()
+
+ def test_inventory_tag_del(self):
+ tags = ("europe", "africa",)
+ tagsf = ("free", )
+ hosts = ("loadbalancer1", "loadbalancer2", "shell2",)
+ hostsf = ("web2", "web3",)
+ tagfile = op.join(fixtures, "tags")
+ with open(tagfile, "w") as f:
+ for x in tagsf:
+ f.write("{}\n".format(x))
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hostsf:
+ f.write("{}\n".format(x))
+ invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+ action="del", tags=tags,
+ tagfile=tagfile, hosts=hosts,
+ hostfile=hostfile)
+ invTag.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts + hostsf)
+ failed = False
+ for host, taglist in invList.entries():
+ if host not in hosts + hostsf:
+ failed = True
+ break
+ for x in tagsf + tags:
+ if x in taglist:
+ failed = True
+ break
+ if failed:
+ break
+ os.remove(tagfile)
+ os.remove(hostfile)
+ if failed:
+ self.fail()
+
+ def test_inventory_tag_del_all_tags(self):
+ hosts = ("web3", "shell1",)
+ hostsf = ("shell2", "loadbalancer1",)
+ hostfile = op.join(fixtures, "hosts")
+ with open(hostfile, "w") as f:
+ for x in hostsf:
+ f.write("{}\n".format(x))
+ invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+ action="del", all=True,
+ hosts=hosts, hostfile=hostfile)
+ invHost.run()
+ invList = inventory.InventoryList(db_basedir=inventory_dir,
+ hosts=hosts + hostsf)
+ for host, htags in invList.entries():
+ self.assertEqual(htags, ())
+ os.remove(hostfile)
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py
index 3e07c1a7..e0da2d9f 100644
--- a/cdist/test/manifest/__init__.py
+++ b/cdist/test/manifest/__init__.py
@@ -53,6 +53,7 @@ class ManifestTestCase(test.CdistTestCase):
base_root_path = os.path.join(out_path, hostdir)
self.local = local.Local(
target_host=self.target_host,
+ target_host_tags=self.target_host_tags,
base_root_path=base_root_path,
host_dir_name=hostdir,
exec_path=cdist.test.cdist_exec_path,
@@ -93,6 +94,8 @@ class ManifestTestCase(test.CdistTestCase):
self.local.type_path)
self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
self.assertEqual(output_dict['__files'], self.local.files_path)
+ self.assertEqual(output_dict['__target_host_tags'],
+ self.local.target_host_tags)
def test_type_manifest_environment(self):
cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
@@ -126,6 +129,8 @@ class ManifestTestCase(test.CdistTestCase):
self.assertEqual(output_dict['__object_id'], cdist_object.object_id)
self.assertEqual(output_dict['__object_name'], cdist_object.name)
self.assertEqual(output_dict['__files'], self.local.files_path)
+ self.assertEqual(output_dict['__target_host_tags'],
+ self.local.target_host_tags)
def test_debug_env_setup(self):
current_level = self.log.getEffectiveLevel()
diff --git a/cdist/test/manifest/fixtures/conf/manifest/dump_environment b/cdist/test/manifest/fixtures/conf/manifest/dump_environment
index 702145e2..9f9df372 100755
--- a/cdist/test/manifest/fixtures/conf/manifest/dump_environment
+++ b/cdist/test/manifest/fixtures/conf/manifest/dump_environment
@@ -9,4 +9,5 @@ __global: $__global
__cdist_type_base_path: $__cdist_type_base_path
__manifest: $__manifest
__files: $__files
+__target_host_tags: $__target_host_tags
DONE
diff --git a/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest b/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest
index 757d07b5..fec5cb3f 100755
--- a/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest
+++ b/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest
@@ -13,4 +13,5 @@ __object: $__object
__object_id: $__object_id
__object_name: $__object_name
__files: $__files
+__target_host_tags: $__target_host_tags
DONE
diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash
index 6b58e2a2..1311384a 100644
--- a/completions/bash/cdist-completion.bash
+++ b/completions/bash/cdist-completion.bash
@@ -5,8 +5,8 @@ _cdist()
cur="${COMP_WORDS[COMP_CWORD]}"
prev="${COMP_WORDS[COMP_CWORD-1]}"
prevprev="${COMP_WORDS[COMP_CWORD-2]}"
- opts="-h --help -d --debug -v --verbose -V --version"
- cmds="banner shell config install"
+ opts="-h --help -q --quiet -v --verbose -V --version"
+ cmds="banner config install inventory shell"
case "${prevprev}" in
shell)
@@ -18,6 +18,41 @@ _cdist()
;;
esac
;;
+ inventory)
+ case "${prev}" in
+ list)
+ opts="-h --help -q --quiet -v --verbose -b --beta \
+ -I --invento/y -a --all -f --file -H --host-only \
+ -t --tag"
+ COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+ return 0
+ ;;
+ add-host)
+ opts="-h --help -q --quiet -v --verbose -b --beta \
+ -I --inventory -f --file"
+ COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+ return 0
+ ;;
+ del-host)
+ opts="-h --help -q --quiet -v --verbose -b --beta \
+ -I --inventory -a --all -f --file"
+ COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+ return 0
+ ;;
+ add-tag)
+ opts="-h --help -q --quiet -v --verbose -b --beta \
+ -I --inventory -f --file -T --tag-file -t --taglist"
+ COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+ return 0
+ ;;
+ del-tag)
+ opts="-h --help -q --quiet -v --verbose -b --beta \
+ -I --inventory -a --all -f --file -T --tag-file -t --taglist"
+ COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+ return 0
+ ;;
+ esac
+ ;;
esac
case "${prev}" in
@@ -26,23 +61,31 @@ _cdist()
return 0
;;
banner)
- opts="-h --help -d --debug -v --verbose"
+ opts="-h --help -q --quiet -v --verbose"
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
;;
shell)
- opts="-h --help -d --debug -v --verbose -s --shell"
+ opts="-h --help -q --quiet -v --verbose -s --shell"
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
;;
config|install)
- opts="-h --help -d --debug -v --verbose -b --beta \
- -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \
- -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential \
- --remote-copy --remote-exec"
+ opts="-h --help -q --quiet -v --verbose -b --beta \
+ -I --inventory -C --cache-path-pattern -c --conf-dir \
+ -f --file -i --initial-manifest -A --all-tagged \
+ -j --jobs -n --dry-run -o --out-dir -p --parallel \
+ -r --remote-out-dir \
+ -s --sequential --remote-copy --remote-exec -t --tag -a --all"
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
return 0
;;
+ inventory)
+ cmds="list add-host del-host add-tag del-tag"
+ opts="-h --help -q --quiet -v --verbose"
+ COMPREPLY=( $(compgen -W "${opts} ${cmds}" -- ${cur}) )
+ return 0
+ ;;
*)
;;
esac
diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist
index 082cad12..405023ed 100644
--- a/completions/zsh/_cdist
+++ b/completions/zsh/_cdist
@@ -11,16 +11,16 @@ _cdist()
case $state in
opts_cmds)
- _arguments '1:Options and commands:(banner config shell install -h --help -d --debug -v --verbose -V --version)'
+ _arguments '1:Options and commands:(banner config install inventory shell -h --help -q --quiet -v --verbose -V --version)'
;;
*)
case $words[2] in
-*)
- opts=(-h --help -d --debug -v --verbose -V --version)
+ opts=(-h --help -q --quiet -v --verbose -V --version)
compadd "$@" -- $opts
;;
banner)
- opts=(-h --help -d --debug -v --verbose)
+ opts=(-h --help -q --quiet -v --verbose)
compadd "$@" -- $opts
;;
shell)
@@ -30,16 +30,45 @@ _cdist()
compadd "$@" -- $shells
;;
*)
- opts=(-h --help -d --debug -v --verbose -s --shell)
+ opts=(-h --help -q --quiet -v --verbose -s --shell)
compadd "$@" -- $opts
;;
esac
;;
config|install)
- opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec)
+ opts=(-h --help -q --quiet -v --verbose -a --all -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec -t --tag -I --inventory -A --all-tagged)
compadd "$@" -- $opts
;;
- *)
+ inventory)
+ case $words[3] in
+ list)
+ opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -H --host-only -t --tag)
+ compadd "$@" -- $opts
+ ;;
+ add-host)
+ opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file)
+ compadd "$@" -- $opts
+ ;;
+ del-host)
+ opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file)
+ compadd "$@" -- $opts
+ ;;
+ add-tag)
+ opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file -T --tag-file -t --taglist)
+ compadd "$@" -- $opts
+ ;;
+ del-tag)
+ opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -T --tag-file -t --taglist)
+ compadd "$@" -- $opts
+ ;;
+ *)
+ cmds=(list add-host del-host add-tag del-tag)
+ opts=(-h --help -q --quiet -v --verbose)
+ compadd "$@" -- $cmds $opts
+ ;;
+ esac
+ ;;
+ *)
;;
esac
esac
diff --git a/docs/changelog b/docs/changelog
index fee1e1ff..b8c58adf 100644
--- a/docs/changelog
+++ b/docs/changelog
@@ -2,6 +2,10 @@ Changelog
---------
next:
+ * Core: Add inventory functionality (Darko Poljak)
+ * Core: Expose inventory host tags in __target_host_tags env var (Darko Poljak)
+
+4.5.0: 2017-07-20
* Types: Fix install types (Steven Armstrong)
* Core: Add -r command line option for setting remote base path (Steven Armstrong)
* Core: Allow manifest and gencode scripts to be written in any language (Darko Poljak)
diff --git a/docs/src/cdist-inventory.rst b/docs/src/cdist-inventory.rst
new file mode 100644
index 00000000..584fe310
--- /dev/null
+++ b/docs/src/cdist-inventory.rst
@@ -0,0 +1,211 @@
+Inventory
+=========
+
+Introduction
+------------
+
+cdist comes with simple built-in tag based inventory. It is a simple inventory
+with list of hosts and a host has a list of tags.
+Inventory functionality is still in **beta** so it can be used only if beta
+command line flag is specified (-b, --beta) or setting CDIST_BETA env var.
+
+Description
+-----------
+
+The idea is to have simple tagging inventory. There is a list of hosts and for
+each host there are tags. Inventory database is a set of files under inventory
+database base directory. Filename equals hostname. Each file contains tags for
+hostname with each tag on its own line.
+
+Using inventory you can now configure hosts by selecting them by tags.
+
+Tags have no values, as tags are just tags. Tag name-value would in this
+context mean that host has two tags and it is selected by specifying that both
+tags are present.
+
+This inventory is **KISS** cdist built-in inventory database. You can maintain it
+using cdist inventory interface or using standard UNIX tools.
+
+cdist inventory interface
+-------------------------
+
+With cdist inventory interface you can list host(s) and tag(s), add host(s),
+add tag(s), delete host(s) and delete tag(s).
+
+Configuring hosts using inventory
+---------------------------------
+
+config command now has new options, **-t**, **-a** and **-A**.
+
+**-A** means that all hosts in tag db is selected.
+
+**-a** means that selected hosts must contain ALL specified tags.
+
+**-t** means that host specifies tag - all hosts that have specified tags are
+selected.
+
+Examples
+--------
+
+.. code-block:: sh
+
+ # List inventory content
+ $ cdist inventory list -b
+
+ # List inventory for specified host localhost
+ $ cdist inventory list -b localhost
+
+ # List inventory for specified tag loadbalancer
+ $ cdist inventory list -b -t loadbalancer
+
+ # Add hosts to inventory
+ $ cdist inventory add-host -b web1 web2 web3
+
+ # Delete hosts from file old-hosts from inventory
+ $ cdist inventory del-host -b -f old-hosts
+
+ # Add tags to specifed hosts
+ $ cdist inventory add-tag -b -t europe,croatia,web,static web1 web2
+
+ # Add tag to all hosts in inventory
+ $ cdist inventory add-tag -b -t vm
+
+ # Delete all tags from specified host
+ $ cdist inventory del-tag -b -a localhost
+
+ # Delete tags read from stdin from hosts specified by file hosts
+ $ cdist inventory del-tag -b -T - -f hosts
+
+ # Configure hosts from inventory with any of specified tags
+ $ cdist config -b -t web dynamic
+
+ # Configure hosts from inventory with all specified tags
+ $ cdist config -b -t -a web dynamic
+
+ # Configure all hosts from inventory db
+ $ cdist config -b -A
+
+Example of manipulating database
+--------------------------------
+
+.. code-block:: sh
+
+ $ python3 scripts/cdist inventory list -b
+ $ python3 scripts/cdist inventory add-host -b localhost
+ $ python3 scripts/cdist inventory add-host -b test.mycloud.net
+ $ python3 scripts/cdist inventory list -b
+ localhost
+ test.mycloud.net
+ $ python3 scripts/cdist inventory add-host -b web1.mycloud.net web2.mycloud.net shell1.mycloud.net shell2.mycloud.net
+ $ python3 scripts/cdist inventory list -b
+ localhost
+ test.mycloud.net
+ web1.mycloud.net
+ web2.mycloud.net
+ shell1.mycloud.net
+ shell2.mycloud.net
+ $ python3 scripts/cdist inventory add-tag -b -t web web1.mycloud.net web2.mycloud.net
+ $ python3 scripts/cdist inventory add-tag -b -t shell shell1.mycloud.net shell2.mycloud.net
+ $ python3 scripts/cdist inventory add-tag -b -t cloud
+ $ python3 scripts/cdist inventory list -b
+ localhost cloud
+ test.mycloud.net cloud
+ web1.mycloud.net cloud,web
+ web2.mycloud.net cloud,web
+ shell1.mycloud.net cloud,shell
+ shell2.mycloud.net cloud,shell
+ $ python3 scripts/cdist inventory add-tag -b -t test,web,shell test.mycloud.net
+ $ python3 scripts/cdist inventory list -b
+ localhost cloud
+ test.mycloud.net cloud,shell,test,web
+ web1.mycloud.net cloud,web
+ web2.mycloud.net cloud,web
+ shell1.mycloud.net cloud,shell
+ shell2.mycloud.net cloud,shell
+ $ python3 scripts/cdist inventory del-tag -b -t shell test.mycloud.net
+ $ python3 scripts/cdist inventory list -b
+ localhost cloud
+ test.mycloud.net cloud,test,web
+ web1.mycloud.net cloud,web
+ web2.mycloud.net cloud,web
+ shell1.mycloud.net cloud,shell
+ shell2.mycloud.net cloud,shell
+ $ python3 scripts/cdist inventory add-tag -b -t all
+ $ python3 scripts/cdist inventory add-tag -b -t mistake
+ $ python3 scripts/cdist inventory list -b
+ localhost all,cloud,mistake
+ test.mycloud.net all,cloud,mistake,test,web
+ web1.mycloud.net all,cloud,mistake,web
+ web2.mycloud.net all,cloud,mistake,web
+ shell1.mycloud.net all,cloud,mistake,shell
+ shell2.mycloud.net all,cloud,mistake,shell
+ $ python3 scripts/cdist inventory del-tag -b -t mistake
+ $ python3 scripts/cdist inventory list -b
+ localhost all,cloud
+ test.mycloud.net all,cloud,test,web
+ web1.mycloud.net all,cloud,web
+ web2.mycloud.net all,cloud,web
+ shell1.mycloud.net all,cloud,shell
+ shell2.mycloud.net all,cloud,shell
+ $ python3 scripts/cdist inventory del-host -b localhost
+ $ python3 scripts/cdist inventory list -b
+ test.mycloud.net all,cloud,test,web
+ web1.mycloud.net all,cloud,web
+ web2.mycloud.net all,cloud,web
+ shell1.mycloud.net all,cloud,shell
+ shell2.mycloud.net all,cloud,shell
+ $ python3 scripts/cdist inventory list -b -t web
+ test.mycloud.net all,cloud,test,web
+ web1.mycloud.net all,cloud,web
+ web2.mycloud.net all,cloud,web
+ $ python3 scripts/cdist inventory list -b -t -a web test
+ test.mycloud.net all,cloud,test,web
+ $ python3 scripts/cdist inventory list -b -t -a web all
+ test.mycloud.net all,cloud,test,web
+ web1.mycloud.net all,cloud,web
+ web2.mycloud.net all,cloud,web
+ $ python3 scripts/cdist inventory list -b -t web all
+ test.mycloud.net all,cloud,test,web
+ web1.mycloud.net all,cloud,web
+ web2.mycloud.net all,cloud,web
+ shell1.mycloud.net all,cloud,shell
+ shell2.mycloud.net all,cloud,shell
+ $ cd cdist/inventory
+ $ ls -1
+ shell1.mycloud.net
+ shell2.mycloud.net
+ test.mycloud.net
+ web1.mycloud.net
+ web2.mycloud.net
+ $ ls -l
+ total 20
+ -rw-r--r-- 1 darko darko 16 Jun 24 12:43 shell1.mycloud.net
+ -rw-r--r-- 1 darko darko 16 Jun 24 12:43 shell2.mycloud.net
+ -rw-r--r-- 1 darko darko 19 Jun 24 12:43 test.mycloud.net
+ -rw-r--r-- 1 darko darko 14 Jun 24 12:43 web1.mycloud.net
+ -rw-r--r-- 1 darko darko 14 Jun 24 12:43 web2.mycloud.net
+ $ cat test.mycloud.net
+ test
+ all
+ web
+ cloud
+ $ cat web2.mycloud.net
+ all
+ web
+ cloud
+
+For more info about inventory commands and options see `cdist <man1/cdist.html>`_\ (1).
+
+Using external inventory
+------------------------
+
+cdist can be used with any external inventory where external inventory is
+some storage or database from which you can get a list of hosts to configure.
+cdist can then be fed with this list of hosts through stdin or file using
+**-f** option. For example, if your host list is stored in sqlite3 database
+hosts.db and you want to select hosts which purpose is **django** then you
+can use it with cdist like:
+
+.. code-block:: sh
+
+ $ sqlite3 hosts.db "select hostname from hosts where purpose = 'django';" | cdist config
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index 5889ded9..89820358 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -63,6 +63,10 @@ cdist/conf/
The distribution configuration directory.
This contains types and explorers to be used.
+cdist/inventory/
+ The distribution inventory directory.
+ This path is relative to cdist installation directory.
+
confdir
Cdist will use all available configuration directories and create
a temporary confdir containing links to the real configuration directories.
@@ -239,6 +243,9 @@ __target_fqdn
This variable is derived from **__target_host**
(using **socket.getfqdn()**).
Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
+__target_host_tags
+ Comma separated list of target host tags.
+ Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
__type
Path to the current type.
Available for: type manifest, type gencode.
@@ -274,6 +281,9 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp).
+CDIST_INVENTORY_DIR
+ Use this directory as inventory directory.
+
CDIST_BETA
Enable beta functionalities.
diff --git a/docs/src/index.rst b/docs/src/index.rst
index b33b707d..42c21199 100644
--- a/docs/src/index.rst
+++ b/docs/src/index.rst
@@ -24,6 +24,7 @@ Contents:
cdist-explorer
cdist-messaging
cdist-parallelization
+ cdist-inventory
cdist-reference
cdist-best-practice
cdist-stages
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 37ea4969..d6bd1c8f 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -11,21 +11,46 @@ SYNOPSIS
::
- cdist [-h] [-q] [-v] [-V] {banner,config,shell,install} ...
+ cdist [-h] [-q] [-v] [-V] {banner,config,install,inventory,shell} ...
cdist banner [-h] [-q] [-v]
cdist config [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
- [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
- [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s]
- [host [host ...]]
+ [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
+ [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
+ [-f HOSTFILE] [-p] [-s] [-t]
+ [host [host ...]]
cdist install [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
- [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
- [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s]
- [host [host ...]]
+ [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
+ [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
+ [-f HOSTFILE] [-p] [-s] [-t]
+ [host [host ...]]
+
+ cdist inventory [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+ {add-host,add-tag,del-host,del-tag,list} ...
+
+ cdist inventory add-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+ [-f HOSTFILE]
+ [host [host ...]]
+
+ cdist inventory add-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+ [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+ [host [host ...]]
+
+ cdist inventory del-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+ [-f HOSTFILE]
+ [host [host ...]]
+
+ cdist inventory del-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+ [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+ [host [host ...]]
+
+ cdist inventory list [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+ [-f HOSTFILE] [-H] [-t]
+ [host [host ...]]
cdist shell [-h] [-q] [-v] [-s SHELL]
@@ -72,6 +97,15 @@ CONFIG/INSTALL
--------------
Configure/install one or more hosts.
+.. option:: -A, --all-tagged
+
+ use all hosts present in tags db
+
+.. option:: -a, --all
+
+ list hosts that have all specified tags, if -t/--tag
+ is specified
+
.. option:: -b, --beta
Enable beta functionality.
@@ -103,6 +137,16 @@ Configure/install one or more hosts.
read hosts from stdin. For the file format see
:strong:`HOSTFILE FORMAT` below.
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+ Use specified custom inventory directory. Inventory
+ directory is set up by the following rules: if this
+ argument is set then specified directory is used, if
+ CDIST_INVENTORY_DIR env var is set then its value is
+ used, if HOME env var is set then ~/.cdit/inventory is
+ used, otherwise distribution inventory directory is
+ used.
+
.. option:: -i MANIFEST, --initial-manifest MANIFEST
Path to a cdist manifest or - to read from stdin
@@ -141,6 +185,10 @@ Configure/install one or more hosts.
Command to use for remote execution (should behave like ssh)
+.. option:: -t, --tag
+
+ host is specified by tag, not hostname/address; list
+ all hosts that contain any of specified tags
HOSTFILE FORMAT
~~~~~~~~~~~~~~~
@@ -174,6 +222,246 @@ Resulting path is used to specify cache path subdirectory under which
current host cache data are saved.
+INVENTORY
+---------
+Manage inventory database.
+Currently in beta with all sub-commands.
+
+
+INVENTORY ADD-HOST
+------------------
+Add host(s) to inventory database.
+
+.. option:: host
+
+ host(s) to add
+
+.. option:: -b, --beta
+
+ Enable beta functionalities. Beta functionalities
+ include inventory command with all sub-commands and
+ all options; config sub-command options: -j/--jobs,
+ -t/--tag, -a/--all.
+
+ Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+ Read additional hosts to add from specified file or
+ from stdin if '-' (each host on separate line). If no
+ host or host file is specified then, by default, read
+ from stdin. Hostfile format is the same as config hostfile format.
+
+.. option:: -h, --help
+
+ show this help message and exit
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+ Use specified custom inventory directory. Inventory
+ directory is set up by the following rules: if this
+ argument is set then specified directory is used, if
+ CDIST_INVENTORY_DIR env var is set then its value is
+ used, if HOME env var is set then ~/.cdist/inventory is
+ used, otherwise distribution inventory directory is
+ used.
+
+
+INVENTORY ADD-TAG
+-----------------
+Add tag(s) to inventory database.
+
+.. option:: host
+
+ list of host(s) for which tags are added
+
+.. option:: -b, --beta
+
+ Enable beta functionalities. Beta functionalities
+ include inventory command with all sub-commands and
+ all options; config sub-command options: -j/--jobs,
+ -t/--tag, -a/--all.
+
+ Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+ Read additional hosts to add tags from specified file
+ or from stdin if '-' (each host on separate line). If
+ no host or host file is specified then, by default,
+ read from stdin. If no tags/tagfile nor hosts/hostfile
+ are specified then tags are read from stdin and are
+ added to all hosts. Hostfile format is the same as config hostfile format.
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+ Use specified custom inventory directory. Inventory
+ directory is set up by the following rules: if this
+ argument is set then specified directory is used, if
+ CDIST_INVENTORY_DIR env var is set then its value is
+ used, if HOME env var is set then ~/.cdist/inventory is
+ used, otherwise distribution inventory directory is
+ used.
+
+.. option:: -T TAGFILE, --tag-file TAGFILE
+
+ Read additional tags to add from specified file or
+ from stdin if '-' (each tag on separate line). If no
+ tag or tag file is specified then, by default, read
+ from stdin. If no tags/tagfile nor hosts/hostfile are
+ specified then tags are read from stdin and are added
+ to all hosts. Tagfile format is the same as config hostfile format.
+
+.. option:: -t TAGLIST, --taglist TAGLIST
+
+ Tag list to be added for specified host(s), comma
+ separated values
+
+
+INVENTORY DEL-HOST
+------------------
+Delete host(s) from inventory database.
+
+.. option:: host
+
+ host(s) to delete
+
+.. option:: -a, --all
+
+ Delete all hosts
+
+.. option:: -b, --beta
+
+ Enable beta functionalities. Beta functionalities
+ include inventory command with all sub-commands and
+ all options; config sub-command options: -j/--jobs,
+ -t/--tag, -a/--all.
+
+ Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+ Read additional hosts to delete from specified file or
+ from stdin if '-' (each host on separate line). If no
+ host or host file is specified then, by default, read
+ from stdin. Hostfile format is the same as config hostfile format.
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+ Use specified custom inventory directory. Inventory
+ directory is set up by the following rules: if this
+ argument is set then specified directory is used, if
+ CDIST_INVENTORY_DIR env var is set then its value is
+ used, if HOME env var is set then ~/.cdist/inventory is
+ used, otherwise distribution inventory directory is
+ used.
+
+
+INVENTORY DEL-TAG
+-----------------
+Delete tag(s) from inventory database.
+
+.. option:: host
+
+ list of host(s) for which tags are deleted
+
+.. option:: -a, --all
+
+ Delete all tags for specified host(s)
+
+.. option:: -b, --beta
+
+ Enable beta functionalities. Beta functionalities
+ include inventory command with all sub-commands and
+ all options; config sub-command options: -j/--jobs,
+ -t/--tag, -a/--all.
+
+ Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+ Read additional hosts to delete tags for from
+ specified file or from stdin if '-' (each host on
+ separate line). If no host or host file is specified
+ then, by default, read from stdin. If no tags/tagfile
+ nor hosts/hostfile are specified then tags are read
+ from stdin and are deleted from all hosts. Hostfile
+ format is the same as config hostfile format.
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+ Use specified custom inventory directory. Inventory
+ directory is set up by the following rules: if this
+ argument is set then specified directory is used, if
+ CDIST_INVENTORY_DIR env var is set then its value is
+ used, if HOME env var is set then ~/.cdist/inventory is
+ used, otherwise distribution inventory directory is
+ used.
+
+.. option:: -T TAGFILE, --tag-file TAGFILE
+
+ Read additional tags from specified file or from stdin
+ if '-' (each tag on separate line). If no tag or tag
+ file is specified then, by default, read from stdin.
+ If no tags/tagfile nor hosts/hostfile are specified
+ then tags are read from stdin and are added to all
+ hosts. Tagfile format is the same as config hostfile format.
+
+.. option:: -t TAGLIST, --taglist TAGLIST
+
+ Tag list to be deleted for specified host(s), comma
+ separated values
+
+
+INVENTORY LIST
+--------------
+List inventory database.
+
+.. option:: host
+
+ host(s) to list
+
+.. option:: -a, --all
+
+ list hosts that have all specified tags, if -t/--tag
+ is specified
+
+.. option:: -b, --beta
+
+ Enable beta functionalities. Beta functionalities
+ include inventory command with all sub-commands and
+ all options; config sub-command options: -j/--jobs,
+ -t/--tag, -a/--all.
+
+ Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+ Read additional hosts to list from specified file or
+ from stdin if '-' (each host on separate line). If no
+ host or host file is specified then, by default, list
+ all. Hostfile format is the same as config hostfile format.
+
+.. option:: -H, --host-only
+
+ Suppress tags listing
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+ Use specified custom inventory directory. Inventory
+ directory is set up by the following rules: if this
+ argument is set then specified directory is used, if
+ CDIST_INVENTORY_DIR env var is set then its value is
+ used, if HOME env var is set then ~/.cdist/inventory is
+ used, otherwise distribution inventory directory is
+ used.
+
+.. option:: -t, --tag
+
+ host is specified by tag, not hostname/address; list
+ all hosts that contain any of specified tags
+
+
SHELL
-----
This command allows you to spawn a shell that enables access
@@ -186,14 +474,21 @@ usage. Its primary use is for debugging type parameters.
Select shell to use, defaults to current shell. Used shell should
be POSIX compatible shell.
+
FILES
-----
~/.cdist
Your personal cdist config directory. If exists it will be
automatically used.
+~/.cdist/inventory
+ The home inventory directory. If ~/.cdist exists it will be used as
+ default inventory directory.
cdist/conf
The distribution configuration directory. It contains official types and
explorers. This path is relative to cdist installation directory.
+cdist/inventory
+ The distribution inventory directory.
+ This path is relative to cdist installation directory.
NOTES
-----
@@ -243,6 +538,43 @@ EXAMPLES
# Install ikq05.ethz.ch with debug enabled
% cdist install -vvv ikq05.ethz.ch
+ # List inventory content
+ % cdist inventory list -b
+
+ # List inventory for specified host localhost
+ % cdist inventory list -b localhost
+
+ # List inventory for specified tag loadbalancer
+ % cdist inventory list -b -t loadbalancer
+
+ # Add hosts to inventory
+ % cdist inventory add-host -b web1 web2 web3
+
+ # Delete hosts from file old-hosts from inventory
+ % cdist inventory del-host -b -f old-hosts
+
+ # Add tags to specifed hosts
+ % cdist inventory add-tag -b -t europe,croatia,web,static web1 web2
+
+ # Add tag to all hosts in inventory
+ % cdist inventory add-tag -b -t vm
+
+ # Delete all tags from specified host
+ % cdist inventory del-tag -b -a localhost
+
+ # Delete tags read from stdin from hosts specified by file hosts
+ % cdist inventory del-tag -b -T - -f hosts
+
+ # Configure hosts from inventory with any of specified tags
+ % cdist config -b -t web dynamic
+
+ # Configure hosts from inventory with all specified tags
+ % cdist config -b -t -a web dynamic
+
+ # Configure all hosts from inventory db
+ $ cdist config -b -A
+
+
ENVIRONMENT
-----------
TMPDIR, TEMP, TMP
@@ -272,6 +604,9 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp).
+CDIST_INVENTORY_DIR
+ Use this directory as inventory directory.
+
CDIST_BETA
Enable beta functionality.
diff --git a/scripts/cdist b/scripts/cdist
index 81220ca3..d9dce35f 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -33,6 +33,7 @@ def commandline():
import cdist.config
import cdist.install
import cdist.shell
+ import cdist.inventory
import shutil
import os