Merge remote-tracking branch 'ungleich/master' into ssh-mux-sigpipe

.gitignore

@@ -11,6 +11,9 @@ docs/src/cdist-reference.rst
 # Ignore cdist cache for version control
 /cache/
 
+# Ignore inventory basedir
+cdist/inventory/
+
 # Python: cache, distutils, distribution in general
 __pycache__/
 *.pyc

cdist/argparse.py

@@ -7,10 +7,10 @@ import collections
 
 
 # set of beta sub-commands
-BETA_COMMANDS = set(('install', ))
+BETA_COMMANDS = set(('install', 'inventory', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
-    'config': set(('jobs', )),
+    'config': set(('jobs', 'tag', 'all_tagged_hosts', )),
 }
 EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
 # Parser others can reuse
@@ -121,6 +121,17 @@ def get_parsers():
             'banner', parents=[parser['loglevel']])
     parser['banner'].set_defaults(func=cdist.banner.banner)
 
+    parser['inventory_common'] = argparse.ArgumentParser(add_help=False)
+    parser['inventory_common'].add_argument(
+           '-I', '--inventory',
+           help=('Use specified custom inventory directory. '
+                 'Inventory directory is set up by the following rules: '
+                 'if this argument is set then specified directory is used, '
+                 'if CDIST_INVENTORY_DIR env var is set then its value is '
+                 'used, if HOME env var is set then ~/.cdist/inventory is '
+                 'used, otherwise distribution inventory directory is used.'),
+           dest="inventory_dir", required=False)
+
     # Config
     parser['config_main'] = argparse.ArgumentParser(add_help=False)
     parser['config_main'].add_argument(
@@ -156,6 +167,10 @@ def get_parsers():
     # remote-copy and remote-exec defaults are environment variables
     # if set; if not then None - these will be futher handled after
     # parsing to determine implementation default
+    parser['config_main'].add_argument(
+           '-r', '--remote-out-dir',
+           help='Directory to save cdist output in on the target host',
+           dest="remote_out_path")
     parser['config_main'].add_argument(
            '--remote-copy',
            help='Command to use for remote copy (should behave like scp)',
@@ -170,6 +185,15 @@ def get_parsers():
 
     # Config
     parser['config_args'] = argparse.ArgumentParser(add_help=False)
+    parser['config_args'].add_argument(
+             '-A', '--all-tagged',
+             help=('use all hosts present in tags db'),
+             action="store_true", dest="all_tagged_hosts", default=False)
+    parser['config_args'].add_argument(
+             '-a', '--all',
+             help=('list hosts that have all specified tags, '
+                   'if -t/--tag is specified'),
+             action="store_true", dest="has_all_tags", default=False)
     parser['config_args'].add_argument(
             'host', nargs='*', help='host(s) to operate on')
     parser['config_args'].add_argument(
@@ -183,17 +207,19 @@ def get_parsers():
            '-p', '--parallel',
            help='operate on multiple hosts in parallel',
            action='store_true', dest='parallel')
-    parser['config_args'].add_argument(
-           '-r', '--remote-out-dir',
-           help='Directory to save cdist output in on the target host',
-           dest="remote_out_path")
     parser['config_args'].add_argument(
            '-s', '--sequential',
            help='operate on multiple hosts sequentially (default)',
            action='store_false', dest='parallel')
+    parser['config_args'].add_argument(
+             '-t', '--tag',
+             help=('host is specified by tag, not hostname/address; '
+                   'list all hosts that contain any of specified tags'),
+             dest='tag', required=False, action="store_true", default=False)
     parser['config'] = parser['sub'].add_parser(
             'config', parents=[parser['loglevel'], parser['beta'],
                                parser['config_main'],
+                               parser['inventory_common'],
                                parser['config_args']])
     parser['config'].set_defaults(func=cdist.config.Config.commandline)
 
@@ -202,6 +228,134 @@ def get_parsers():
                                                  parents=[parser['config']])
     parser['install'].set_defaults(func=cdist.install.Install.commandline)
 
+    # Inventory
+    parser['inventory'] = parser['sub'].add_parser(
+           'inventory', parents=[parser['loglevel'], parser['beta'],
+                                 parser['inventory_common']])
+    parser['invsub'] = parser['inventory'].add_subparsers(
+            title="Inventory commands", dest="subcommand")
+
+    parser['add-host'] = parser['invsub'].add_parser(
+            'add-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['inventory_common']])
+    parser['add-host'].add_argument(
+            'host', nargs='*', help='host(s) to add')
+    parser['add-host'].add_argument(
+           '-f', '--file',
+           help=('Read additional hosts to add from specified file '
+                 'or from stdin if \'-\' (each host on separate line). '
+                 'If no host or host file is specified then, by default, '
+                 'read from stdin.'),
+           dest='hostfile', required=False)
+
+    parser['add-tag'] = parser['invsub'].add_parser(
+            'add-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['inventory_common']])
+    parser['add-tag'].add_argument(
+           'host', nargs='*',
+           help='list of host(s) for which tags are added')
+    parser['add-tag'].add_argument(
+           '-f', '--file',
+           help=('Read additional hosts to add tags from specified file '
+                 'or from stdin if \'-\' (each host on separate line). '
+                 'If no host or host file is specified then, by default, '
+                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                 ' are specified then tags are read from stdin and are'
+                 ' added to all hosts.'),
+           dest='hostfile', required=False)
+    parser['add-tag'].add_argument(
+           '-T', '--tag-file',
+           help=('Read additional tags to add from specified file '
+                 'or from stdin if \'-\' (each tag on separate line). '
+                 'If no tag or tag file is specified then, by default, '
+                 'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                 ' are specified then tags are read from stdin and are'
+                 ' added to all hosts.'),
+           dest='tagfile', required=False)
+    parser['add-tag'].add_argument(
+           '-t', '--taglist',
+           help=("Tag list to be added for specified host(s), comma separated"
+                 " values"),
+           dest="taglist", required=False)
+
+    parser['del-host'] = parser['invsub'].add_parser(
+            'del-host', parents=[parser['loglevel'], parser['beta'],
+                                 parser['inventory_common']])
+    parser['del-host'].add_argument(
+            'host', nargs='*', help='host(s) to delete')
+    parser['del-host'].add_argument(
+            '-a', '--all', help=('Delete all hosts'),
+            dest='all', required=False, action="store_true", default=False)
+    parser['del-host'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to delete from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read from stdin.'),
+            dest='hostfile', required=False)
+
+    parser['del-tag'] = parser['invsub'].add_parser(
+            'del-tag', parents=[parser['loglevel'], parser['beta'],
+                                parser['inventory_common']])
+    parser['del-tag'].add_argument(
+            'host', nargs='*',
+            help='list of host(s) for which tags are deleted')
+    parser['del-tag'].add_argument(
+            '-a', '--all',
+            help=('Delete all tags for specified host(s)'),
+            dest='all', required=False, action="store_true", default=False)
+    parser['del-tag'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to delete tags for from specified '
+                  'file or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read from stdin. If no tags/tagfile nor hosts/hostfile'
+                  ' are specified then tags are read from stdin and are'
+                  ' deleted from all hosts.'),
+            dest='hostfile', required=False)
+    parser['del-tag'].add_argument(
+            '-T', '--tag-file',
+            help=('Read additional tags from specified file '
+                  'or from stdin if \'-\' (each tag on separate line). '
+                  'If no tag or tag file is specified then, by default, '
+                  'read from stdin. If no tags/tagfile nor'
+                  ' hosts/hostfile are specified then tags are read from'
+                  ' stdin and are added to all hosts.'),
+            dest='tagfile', required=False)
+    parser['del-tag'].add_argument(
+            '-t', '--taglist',
+            help=("Tag list to be deleted for specified host(s), "
+                  "comma separated values"),
+            dest="taglist", required=False)
+
+    parser['list'] = parser['invsub'].add_parser(
+            'list', parents=[parser['loglevel'], parser['beta'],
+                             parser['inventory_common']])
+    parser['list'].add_argument(
+            'host', nargs='*', help='host(s) to list')
+    parser['list'].add_argument(
+            '-a', '--all',
+            help=('list hosts that have all specified tags, '
+                  'if -t/--tag is specified'),
+            action="store_true", dest="has_all_tags", default=False)
+    parser['list'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to list from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'list all.'), dest='hostfile', required=False)
+    parser['list'].add_argument(
+            '-H', '--host-only', help=('Suppress tags listing'),
+            action="store_true", dest="list_only_host", default=False)
+    parser['list'].add_argument(
+            '-t', '--tag',
+            help=('host is specified by tag, not hostname/address; '
+                  'list all hosts that contain any of specified tags'),
+            action="store_true", default=False)
+
+    parser['inventory'].set_defaults(
+            func=cdist.inventory.Inventory.commandline)
+
     # Shell
     parser['shell'] = parser['sub'].add_parser(
             'shell', parents=[parser['loglevel']])

cdist/conf/type/__install_stage/man.rst

@@ -17,9 +17,9 @@ REQUIRED PARAMETERS
 uri
    The uri from which to fetch the tarball.
    Can be anything understood by curl, e.g:
-     | http://path/to/stage.tgz
+       | http://path/to/stage.tgz
-     | tftp:///path/to/stage.tgz
+       | tftp:///path/to/stage.tgz
-     | file:///local/path/stage.tgz
+       | file:///local/path/stage.tgz
 
 
 OPTIONAL PARAMETERS

cdist/conf/type/__timezone/explorer/timezone_is

@@ -0,0 +1,21 @@
+#!/bin/sh -e
+#
+# 2017 Ander Punnar (cdist at kvlt.ee)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+
+[ -f /etc/timezone ] && cat /etc/timezone

cdist/conf/type/__timezone/gencode-remote

@@ -20,11 +20,16 @@
 #
 # This type allows to configure the desired localtime timezone.
 
-timezone="$__object_id"
+timezone_is=$(cat "$__object/explorer/timezone_is")
+timezone_should="$__object_id"
 os=$(cat "$__global/explorer/os")
 
+if [ "$timezone_is" = "$timezone_should" ]; then
+    exit 0
+fi
+
 case "$os" in
    ubuntu|debian|devuan)
-      echo "echo \"$timezone\" > /etc/timezone"
+      echo "echo \"$timezone_should\" > /etc/timezone"
    ;;
 esac

cdist/config.py

@@ -38,6 +38,9 @@ import cdist.hostsource
 
 import cdist.exec.local
 import cdist.exec.remote
+
+from cdist import inventory
+
 import cdist.util.ipaddr as ipaddr
 
 from cdist import core
@@ -141,11 +144,42 @@ class Config(object):
         base_root_path = cls.create_base_root_path(args.out_path)
 
         hostcnt = 0
-        for host in itertools.chain(cls.hosts(args.host),
+
-                                    cls.hosts(args.hostfile)):
+        if args.tag or args.all_tagged_hosts:
+            inventory.determine_default_inventory_dir(args)
+            if args.all_tagged_hosts:
+                inv_list = inventory.InventoryList(
+                    hosts=None, istag=True, hostfile=None,
+                    db_basedir=args.inventory_dir)
+            else:
+                inv_list = inventory.InventoryList(
+                    hosts=args.host, istag=True, hostfile=args.hostfile,
+                    db_basedir=args.inventory_dir,
+                    has_all_tags=args.has_all_tags)
+            it = inv_list.entries()
+        else:
+            it = itertools.chain(cls.hosts(args.host),
+                                 cls.hosts(args.hostfile))
+        for entry in it:
+            if isinstance(entry, tuple):
+                # if configuring by specified tags
+                host = entry[0]
+                host_tags = entry[1]
+            else:
+                # if configuring by host then check inventory for tags
+                host = entry
+                inventory.determine_default_inventory_dir(args)
+                inv_list = inventory.InventoryList(
+                    hosts=(host,), db_basedir=args.inventory_dir)
+                inv = tuple(inv_list.entries())
+                if inv:
+                    # host is present in inventory and has tags
+                    host_tags = inv[0][1]
+                else:
+                    # host is not present in inventory or has no tags
+                    host_tags = None
             host_base_path, hostdir = cls.create_host_base_dirs(
                 host, base_root_path)
-
             log.debug("Base root path for target host \"{}\" is \"{}\"".format(
                 host, host_base_path))
 
@@ -154,11 +188,12 @@ class Config(object):
                 log.trace("Creating child process for %s", host)
                 process[host] = multiprocessing.Process(
                         target=cls.onehost,
-                        args=(host, host_base_path, hostdir, args, True))
+                        args=(host, host_tags, host_base_path, hostdir, args,
+                              True))
                 process[host].start()
             else:
                 try:
-                    cls.onehost(host, host_base_path, hostdir,
+                    cls.onehost(host, host_tags, host_base_path, hostdir,
                                 args, parallel=False)
                 except cdist.Error as e:
                     failed_hosts.append(host)
@@ -211,7 +246,8 @@ class Config(object):
         return (remote_exec, remote_copy, remote_cmds_cleanup, )
 
     @classmethod
-    def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
+    def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
+                parallel):
         """Configure ONE system"""
 
         log = logging.getLogger(host)
@@ -229,6 +265,7 @@ class Config(object):
 
             local = cdist.exec.local.Local(
                 target_host=target_host,
+                target_host_tags=host_tags,
                 base_root_path=host_base_path,
                 host_dir_name=host_dir_name,
                 initial_manifest=args.manifest,
@@ -445,7 +482,7 @@ class Config(object):
                 for chunk in cargo:
                     for obj in chunk:
                         if (obj.cdist_type == cdist_object.cdist_type and
-                            cdist_object.cdist_type.is_nonparallel):
+                           cdist_object.cdist_type.is_nonparallel):
                             break
                     else:
                         chunk.append(cdist_object)

cdist/core/code.py

@@ -56,6 +56,7 @@ gencode-local
         __object_fq: full qualified object id, iow: $type.name + / + object_id
         __type: full qualified path to the type's dir
         __files: full qualified path to the files dir
+        __target_host_tags: comma spearated list of host tags
 
     returns: string containing the generated code or None
 
@@ -74,6 +75,7 @@ gencode-remote
         __object_fq: full qualified object id, iow: $type.name + / + object_id
         __type: full qualified path to the type's dir
         __files: full qualified path to the files dir
+        __target_host_tags: comma spearated list of host tags
 
     returns: string containing the generated code or None
 
@@ -106,6 +108,7 @@ class Code(object):
             '__target_fqdn': self.target_host[2],
             '__global': self.local.base_path,
             '__files': self.local.files_path,
+            '__target_host_tags': self.local.target_host_tags,
         }
 
     def _run_gencode(self, cdist_object, which):

cdist/core/explorer.py

@@ -77,6 +77,7 @@ class Explorer(object):
             '__target_hostname': self.target_host[1],
             '__target_fqdn': self.target_host[2],
             '__explorer': self.remote.global_explorer_path,
+            '__target_host_tags': self.local.target_host_tags,
         }
         self._type_explorers_transferred = []
         self.jobs = jobs

cdist/core/manifest.py

@@ -42,6 +42,7 @@ common:
                                 types are defined for use in type emulator
             == local.type_path
         __files: full qualified path to the files dir
+        __target_host_tags: comma spearated list of host tags
 
 initial manifest is:
     script: full qualified path to the initial manifest
@@ -109,6 +110,7 @@ class Manifest(object):
             '__target_hostname': self.target_host[1],
             '__target_fqdn': self.target_host[2],
             '__files': self.local.files_path,
+            '__target_host_tags': self.local.target_host_tags,
         }
 
         if self.log.getEffectiveLevel() == logging.DEBUG:

cdist/exec/local.py

@@ -49,6 +49,7 @@ class Local(object):
     """
     def __init__(self,
                  target_host,
+                 target_host_tags,
                  base_root_path,
                  host_dir_name,
                  exec_path=sys.argv[0],
@@ -58,6 +59,10 @@ class Local(object):
                  quiet_mode=False):
 
         self.target_host = target_host
+        if target_host_tags is None:
+            self.target_host_tags = ""
+        else:
+            self.target_host_tags = ",".join(target_host_tags)
         self.hostdir = host_dir_name
         self.base_path = os.path.join(base_root_path, "data")
 

cdist/inventory.py

@@ -0,0 +1,390 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import cdist
+import logging
+import os
+import os.path
+import itertools
+import sys
+from cdist.hostsource import hostfile_process_line
+
+DIST_INVENTORY_DB_NAME = "inventory"
+
+dist_inventory_db = os.path.abspath(os.path.join(
+    os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME))
+
+
+def determine_default_inventory_dir(args):
+    # The order of inventory dir setting by decreasing priority
+    # 1. inventory_dir argument
+    # 2. CDIST_INVENTORY_DIR env var if set
+    # 3. ~/.cdist/inventory if HOME env var is set
+    # 4. distribution inventory directory
+    if not args.inventory_dir:
+        if 'CDIST_INVENTORY_DIR' in os.environ:
+            args.inventory_dir = os.environ['CDIST_INVENTORY_DIR']
+        else:
+            home = cdist.home_dir()
+            if home:
+                args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
+            else:
+                args.inventory_dir = dist_inventory_db
+
+
+def contains_all(big, little):
+    """Return True if big contains all elements from little,
+       False otherwise.
+    """
+    return set(little).issubset(set(big))
+
+
+def contains_any(big, little):
+    """Return True if big contains any element from little,
+       False otherwise.
+    """
+    for x in little:
+        if x in big:
+            return True
+    return False
+
+
+def check_always_true(x, y):
+    return True
+
+
+def rstrip_nl(s):
+    '''str.rstrip "\n" from s'''
+    return str.rstrip(s, "\n")
+
+
+class Inventory(object):
+    """Inventory main class"""
+
+    def __init__(self, db_basedir=dist_inventory_db):
+        self.db_basedir = db_basedir
+        self.log = logging.getLogger("inventory")
+        self.init_db()
+
+    def init_db(self):
+        self.log.debug("Init db: {}".format(self.db_basedir))
+        if not os.path.exists(self.db_basedir):
+            os.makedirs(self.db_basedir, exist_ok=True)
+        elif not os.path.isdir(self.db_basedir):
+            raise cdist.Error(("Invalid inventory db basedir \'{}\',"
+                               " must be a directory").format(self.db_basedir))
+
+    @staticmethod
+    def strlist_to_list(slist):
+        if slist:
+            result = [x for x in slist.split(',') if x]
+        else:
+            result = []
+        return result
+
+    def _input_values(self, source):
+        """Yield input values from source.
+           Source can be a sequence or filename (stdin if '-').
+           In case of filename each line represents one input value.
+        """
+        if isinstance(source, str):
+            import fileinput
+            try:
+                with fileinput.FileInput(files=(source)) as f:
+                    for x in f:
+                        result = hostfile_process_line(x, strip_func=rstrip_nl)
+                        if result:
+                            yield result
+            except (IOError, OSError) as e:
+                raise cdist.Error("Error reading from \'{}\'".format(
+                    source))
+        else:
+            if source:
+                for x in source:
+                    if x:
+                        yield x
+
+    def _host_path(self, host):
+        hostpath = os.path.join(self.db_basedir, host)
+        return hostpath
+
+    def _all_hosts(self):
+        return os.listdir(self.db_basedir)
+
+    def _check_host(self, hostpath):
+        if not os.path.exists(hostpath):
+            return False
+        else:
+            if not os.path.isfile(hostpath):
+                raise cdist.Error(("Host path \'{}\' exists, but is not"
+                                   " a valid file").format(hostpath))
+        return True
+
+    def _read_host_tags(self, hostpath):
+        result = set()
+        with open(hostpath, "rt") as f:
+            for tag in f:
+                tag = tag.rstrip("\n")
+                if tag:
+                    result.add(tag)
+        return result
+
+    def _get_host_tags(self, host):
+        hostpath = self._host_path(host)
+        if self._check_host(hostpath):
+            return self._read_host_tags(hostpath)
+        else:
+            return None
+
+    def _write_host_tags(self, host, tags):
+        hostpath = self._host_path(host)
+        if self._check_host(hostpath):
+            with open(hostpath, "wt") as f:
+                for tag in tags:
+                    f.write("{}\n".format(tag))
+            return True
+        else:
+            return False
+
+    @classmethod
+    def commandline(cls, args):
+        """Manipulate inventory db"""
+        log = logging.getLogger("cdist")
+        if 'taglist' in args:
+            args.taglist = cls.strlist_to_list(args.taglist)
+        determine_default_inventory_dir(args)
+
+        log.info("Using inventory: {}".format(args.inventory_dir))
+        log.debug("Inventory args: {}".format(vars(args)))
+        log.debug("Inventory command: {}".format(args.subcommand))
+
+        if args.subcommand == "list":
+            c = InventoryList(hosts=args.host, istag=args.tag,
+                              hostfile=args.hostfile,
+                              db_basedir=args.inventory_dir,
+                              list_only_host=args.list_only_host,
+                              has_all_tags=args.has_all_tags)
+        elif args.subcommand == "add-host":
+            c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
+                              db_basedir=args.inventory_dir)
+        elif args.subcommand == "del-host":
+            c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
+                              all=args.all, db_basedir=args.inventory_dir,
+                              action="del")
+        elif args.subcommand == "add-tag":
+            c = InventoryTag(hosts=args.host, tags=args.taglist,
+                             hostfile=args.hostfile, tagfile=args.tagfile,
+                             db_basedir=args.inventory_dir)
+        elif args.subcommand == "del-tag":
+            c = InventoryTag(hosts=args.host, tags=args.taglist,
+                             hostfile=args.hostfile, tagfile=args.tagfile,
+                             all=args.all, db_basedir=args.inventory_dir,
+                             action="del")
+        else:
+            raise cdist.Error("Unknown inventory command \'{}\'".format(
+                        args.subcommand))
+        c.run()
+
+
+class InventoryList(Inventory):
+    def __init__(self, hosts=None, istag=False, hostfile=None,
+                 list_only_host=False, has_all_tags=False,
+                 db_basedir=dist_inventory_db):
+        super().__init__(db_basedir)
+        self.hosts = hosts
+        self.istag = istag
+        self.hostfile = hostfile
+        self.list_only_host = list_only_host
+        self.has_all_tags = has_all_tags
+
+    def _print(self, host, tags):
+        if self.list_only_host:
+            print("{}".format(host))
+        else:
+            print("{} {}".format(host, ",".join(sorted(tags))))
+
+    def _do_list(self, it_tags, it_hosts, check_func):
+        if (it_tags is not None):
+            param_tags = set(it_tags)
+            self.log.debug("param_tags: {}".format(param_tags))
+        else:
+            param_tags = set()
+        for host in it_hosts:
+            self.log.debug("host: {}".format(host))
+            tags = self._get_host_tags(host)
+            if tags is None:
+                self.log.info("Host \'{}\' not found, skipped".format(host))
+                continue
+            self.log.debug("tags: {}".format(tags))
+            if check_func(tags, param_tags):
+                yield host, tags
+
+    def entries(self):
+        if not self.hosts and not self.hostfile:
+            self.log.info("Listing all hosts")
+            it_hosts = self._all_hosts()
+            it_tags = None
+            check_func = check_always_true
+        else:
+            it = itertools.chain(self._input_values(self.hosts),
+                                 self._input_values(self.hostfile))
+            if self.istag:
+                self.log.info("Listing by tag(s)")
+                it_hosts = self._all_hosts()
+                it_tags = it
+                if self.has_all_tags:
+                    check_func = contains_all
+                else:
+                    check_func = contains_any
+            else:
+                self.log.info("Listing by host(s)")
+                it_hosts = it
+                it_tags = None
+                check_func = check_always_true
+        for host, tags in self._do_list(it_tags, it_hosts, check_func):
+            yield host, tags
+
+    def host_entries(self):
+        for host, tags in self.entries():
+            yield host
+
+    def run(self):
+        for host, tags in self.entries():
+            self._print(host, tags)
+
+
+class InventoryHost(Inventory):
+    def __init__(self, hosts=None, hostfile=None,
+                 db_basedir=dist_inventory_db, all=False, action="add"):
+        super().__init__(db_basedir)
+        self.actions = ("add", "del")
+        if action not in self.actions:
+            raise cdist.Error("Invalid action \'{}\', valid actions are:"
+                              " {}\n".format(action, self.actions.keys()))
+        self.action = action
+        self.hosts = hosts
+        self.hostfile = hostfile
+        self.all = all
+
+        if not self.hosts and not self.hostfile:
+            self.hostfile = "-"
+
+    def _new_hostpath(self, hostpath):
+        # create empty file
+        with open(hostpath, "w"):
+            pass
+
+    def _action(self, host):
+        if self.action == "add":
+            self.log.info("Adding host \'{}\'".format(host))
+        elif self.action == "del":
+            self.log.info("Deleting host \'{}\'".format(host))
+        hostpath = self._host_path(host)
+        self.log.debug("hostpath: {}".format(hostpath))
+        if self.action == "add" and not os.path.exists(hostpath):
+                self._new_hostpath(hostpath)
+        else:
+            if not os.path.isfile(hostpath):
+                raise cdist.Error(("Host path \'{}\' is"
+                                   " not a valid file").format(hostpath))
+            if self.action == "del":
+                os.remove(hostpath)
+
+    def run(self):
+        if self.action == "del" and self.all:
+            self.log.debug("Doing for all hosts")
+            it = self._all_hosts()
+        else:
+            self.log.debug("Doing for specified hosts")
+            it = itertools.chain(self._input_values(self.hosts),
+                                 self._input_values(self.hostfile))
+        for host in it:
+            self._action(host)
+
+
+class InventoryTag(Inventory):
+    def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None,
+                 db_basedir=dist_inventory_db, all=False, action="add"):
+        super().__init__(db_basedir)
+        self.actions = ("add", "del")
+        if action not in self.actions:
+            raise cdist.Error("Invalid action \'{}\', valid actions are:"
+                              " {}\n".format(action, self.actions.keys()))
+        self.action = action
+        self.hosts = hosts
+        self.tags = tags
+        self.hostfile = hostfile
+        self.tagfile = tagfile
+        self.all = all
+
+        if not self.hosts and not self.hostfile:
+            self.allhosts = True
+        else:
+            self.allhosts = False
+        if not self.tags and not self.tagfile:
+            self.tagfile = "-"
+
+        if self.hostfile == "-" and self.tagfile == "-":
+            raise cdist.Error("Cannot read both, hosts and tags, from stdin")
+
+    def _read_input_tags(self):
+        self.input_tags = set()
+        for tag in itertools.chain(self._input_values(self.tags),
+                                   self._input_values(self.tagfile)):
+            self.input_tags.add(tag)
+
+    def _action(self, host):
+        host_tags = self._get_host_tags(host)
+        if host_tags is None:
+            print("Host \'{}\' does not exist, skipping".format(host),
+                  file=sys.stderr)
+            return
+        self.log.debug("existing host_tags: {}".format(host_tags))
+        if self.action == "del" and self.all:
+            host_tags = set()
+        else:
+            for tag in self.input_tags:
+                if self.action == "add":
+                    self.log.info("Adding tag \'{}\' for host \'{}\'".format(
+                        tag, host))
+                    host_tags.add(tag)
+                elif self.action == "del":
+                    self.log.info("Deleting tag \'{}\' for host \'{}\'".format(
+                        tag, host))
+                    if tag in host_tags:
+                        host_tags.remove(tag)
+        self.log.debug("new host tags: {}".format(host_tags))
+        if not self._write_host_tags(host, host_tags):
+            self.log.info("{} does not exist, skipped".format(host))
+
+    def run(self):
+        if self.allhosts:
+            self.log.debug("Doing for all hosts")
+            it = self._all_hosts()
+        else:
+            self.log.debug("Doing for specified hosts")
+            it = itertools.chain(self._input_values(self.hosts),
+                                 self._input_values(self.hostfile))
+        if not(self.action == "del" and self.all):
+            self._read_input_tags()
+        for host in it:
+            self._action(host)

cdist/shell.py

@@ -44,6 +44,7 @@ class Shell(object):
             "cdist-shell-no-target-host",
             "cdist-shell-no-target-host",
         )
+        self.target_host_tags = ""
 
         host_dir_name = cdist.str_hash(self.target_host[0])
         base_root_path = tempfile.mkdtemp()
@@ -51,6 +52,7 @@ class Shell(object):
 
         self.local = cdist.exec.local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=host_dir_name)
 
@@ -77,6 +79,7 @@ class Shell(object):
             '__manifest': self.local.manifest_path,
             '__explorer': self.local.global_explorer_path,
             '__files': self.local.files_path,
+            '__target_host_tags': self.local.target_host_tags,
         }
 
         self.env.update(additional_env)

cdist/test/__init__.py

@@ -42,6 +42,7 @@ class CdistTestCase(unittest.TestCase):
         'cdisttesthost',
         'cdisttesthost',
     )
+    target_host_tags = "tag1,tag2,tag3"
 
     def mkdtemp(self, **kwargs):
         return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)

cdist/test/code/__init__.py

@@ -46,6 +46,7 @@ class CodeTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=cdist.test.cdist_exec_path,
@@ -97,6 +98,8 @@ class CodeTestCase(test.CdistTestCase):
                          self.cdist_object.object_id)
         self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
         self.assertEqual(output_dict['__files'], self.local.files_path)
+        self.assertEqual(output_dict['__target_host_tags'],
+                         self.local.target_host_tags)
 
     def test_run_gencode_remote_environment(self):
         output_string = self.code.run_gencode_remote(self.cdist_object)
@@ -120,6 +123,8 @@ class CodeTestCase(test.CdistTestCase):
                          self.cdist_object.object_id)
         self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
         self.assertEqual(output_dict['__files'], self.local.files_path)
+        self.assertEqual(output_dict['__target_host_tags'],
+                         self.local.target_host_tags)
 
     def test_transfer_code_remote(self):
         self.cdist_object.code_remote = self.code.run_gencode_remote(

cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local

@@ -9,3 +9,4 @@ echo "echo __object: $__object"
 echo "echo __object_id: $__object_id"
 echo "echo __object_name: $__object_name"
 echo "echo __files: $__files"
+echo "echo __target_host_tags: $__target_host_tags"

cdist/test/config/__init__.py

@@ -60,6 +60,7 @@ class ConfigRunTestCase(test.CdistTestCase):
         os.makedirs(self.host_base_path)
         self.local = cdist.exec.local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir)
 
@@ -164,6 +165,7 @@ class ConfigRunTestCase(test.CdistTestCase):
         """Test if the dryrun option is working like expected"""
         drylocal = cdist.exec.local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             # exec_path can not derivated from sys.argv in case of unittest
@@ -181,6 +183,7 @@ class ConfigRunTestCase(test.CdistTestCase):
         """Test to show dependency resolver warning message."""
         local = cdist.exec.local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=self.host_base_path,
             host_dir_name=self.hostdir,
             exec_path=os.path.abspath(os.path.join(

cdist/test/emulator/__init__.py

@@ -53,6 +53,7 @@ class EmulatorTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
@@ -156,6 +157,7 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
@@ -246,6 +248,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
@@ -279,6 +282,7 @@ class OverrideTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
@@ -322,6 +326,7 @@ class ArgumentsTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
@@ -445,6 +450,7 @@ class StdinTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
@@ -511,6 +517,7 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=host_base_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,

cdist/test/explorer/__init__.py

@@ -50,6 +50,7 @@ class ExplorerClassTestCase(test.CdistTestCase):
 
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=base_root_path,
             host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,

cdist/test/inventory/__init__.py

@@ -0,0 +1,476 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import shutil
+import cdist
+import os.path as op
+import unittest
+import sys
+from cdist import test
+from cdist import inventory
+from io import StringIO
+
+my_dir = op.abspath(op.dirname(__file__))
+fixtures = op.join(my_dir, 'fixtures')
+inventory_dir = op.join(fixtures, "inventory")
+
+
+class InventoryTestCase(test.CdistTestCase):
+
+    def _create_host_with_tags(self, host, tags):
+        os.makedirs(inventory_dir, exist_ok=True)
+        hostfile = op.join(inventory_dir, host)
+        with open(hostfile, "w") as f:
+            for x in tags:
+                f.write("{}\n".format(x))
+
+    def setUp(self):
+        self.maxDiff = None
+        self.db = {
+            "loadbalancer1": ["loadbalancer", "all", "europe", ],
+            "loadbalancer2": ["loadbalancer", "all", "europe", ],
+            "loadbalancer3": ["loadbalancer", "all", "africa", ],
+            "loadbalancer4": ["loadbalancer", "all", "africa", ],
+            "web1": ["web", "all", "static", ],
+            "web2": ["web", "all", "dynamic", ],
+            "web3": ["web", "all", "dynamic", ],
+            "shell1": ["shell", "all", "free", ],
+            "shell2": ["shell", "all", "free", ],
+            "shell3": ["shell", "all", "charge", ],
+            "shell4": ["shell", "all", "charge", ],
+            "monty": ["web", "python", "shell", ],
+            "python": ["web", "python", "shell", ],
+        }
+        for x in self.db:
+            self.db[x] = sorted(self.db[x])
+        for host in self.db:
+            self._create_host_with_tags(host, self.db[host])
+        self.sys_stdout = sys.stdout
+        out = StringIO()
+        sys.stdout = out
+
+    def _get_output(self):
+        sys.stdout.flush()
+        output = sys.stdout.getvalue().strip()
+        return output
+
+    def tearDown(self):
+        sys.stdout = self.sys_stdout
+        shutil.rmtree(inventory_dir)
+
+    def test_inventory_create_db(self):
+        dbdir = op.join(fixtures, "foo")
+        inv = inventory.Inventory(db_basedir=dbdir)
+        self.assertTrue(os.path.isdir(dbdir))
+        self.assertEqual(inv.db_basedir, dbdir)
+        shutil.rmtree(inv.db_basedir)
+
+    # InventoryList
+    def test_inventory_list_print(self):
+        invList = inventory.InventoryList(db_basedir=inventory_dir)
+        invList.run()
+        output = self._get_output()
+        self.assertTrue(' ' in output)
+
+    def test_inventory_list_print_host_only(self):
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          list_only_host=True)
+        invList.run()
+        output = self._get_output()
+        self.assertFalse(' ' in output)
+
+    def test_inventory_list_all(self):
+        invList = inventory.InventoryList(db_basedir=inventory_dir)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        self.assertEqual(db, self.db)
+
+    def test_inventory_list_by_host_hosts(self):
+        hosts = ("web1", "web2", "web3",)
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        expected_db = {host: sorted(self.db[host]) for host in hosts}
+        self.assertEqual(db, expected_db)
+
+    def test_inventory_list_by_host_hostfile(self):
+        hosts = ("web1", "web2", "web3",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hosts:
+                f.write("{}\n".format(x))
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hostfile=hostfile)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        expected_db = {host: sorted(self.db[host]) for host in hosts}
+        self.assertEqual(db, expected_db)
+        os.remove(hostfile)
+
+    def test_inventory_list_by_host_hosts_hostfile(self):
+        hosts = ("shell1", "shell4",)
+        hostsf = ("web1", "web2", "web3",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hostsf:
+                f.write("{}\n".format(x))
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts, hostfile=hostfile)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        import itertools
+        expected_db = {host: sorted(self.db[host]) for host in
+                       itertools.chain(hostsf, hosts)}
+        self.assertEqual(db, expected_db)
+        os.remove(hostfile)
+
+    def _gen_expected_db_for_tags(self, tags):
+        db = {}
+        for host in self.db:
+            for tag in tags:
+                if tag in self.db[host]:
+                    db[host] = self.db[host]
+                    break
+        return db
+
+    def _gen_expected_db_for_has_all_tags(self, tags):
+        db = {}
+        for host in self.db:
+            if set(tags).issubset(set(self.db[host])):
+                db[host] = self.db[host]
+        return db
+
+    def test_inventory_list_by_tag_hosts(self):
+        tags = ("web", "shell",)
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          istag=True, hosts=tags)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        expected_db = self._gen_expected_db_for_tags(tags)
+        self.assertEqual(db, expected_db)
+
+    def test_inventory_list_by_tag_hostfile(self):
+        tags = ("web", "shell",)
+        tagfile = op.join(fixtures, "tags")
+        with open(tagfile, "w") as f:
+            for x in tags:
+                f.write("{}\n".format(x))
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          istag=True, hostfile=tagfile)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        expected_db = self._gen_expected_db_for_tags(tags)
+        self.assertEqual(db, expected_db)
+        os.remove(tagfile)
+
+    def test_inventory_list_by_tag_hosts_hostfile(self):
+        tags = ("web", "shell",)
+        tagsf = ("dynamic", "europe",)
+        tagfile = op.join(fixtures, "tags")
+        with open(tagfile, "w") as f:
+            for x in tagsf:
+                f.write("{}\n".format(x))
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          istag=True, hosts=tags,
+                                          hostfile=tagfile)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        import itertools
+        expected_db = self._gen_expected_db_for_tags(tags + tagsf)
+        self.assertEqual(db, expected_db)
+        os.remove(tagfile)
+
+    def test_inventory_list_by_tag_has_all_tags(self):
+        tags = ("web", "python", "shell",)
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          istag=True, hosts=tags,
+                                          has_all_tags=True)
+        entries = invList.entries()
+        db = {host: sorted(tags) for host, tags in entries}
+        expected_db = self._gen_expected_db_for_has_all_tags(tags)
+        self.assertEqual(db, expected_db)
+
+    # InventoryHost
+    def test_inventory_host_add_hosts(self):
+        hosts = ("spam", "eggs", "foo",)
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="add", hosts=hosts)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir)
+        expected_hosts = tuple(x for x in invList.host_entries() if x in hosts)
+        self.assertEqual(sorted(hosts), sorted(expected_hosts))
+
+    def test_inventory_host_add_hostfile(self):
+        hosts = ("spam-new", "eggs-new", "foo-new",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hosts:
+                f.write("{}\n".format(x))
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="add", hostfile=hostfile)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir)
+        expected_hosts = tuple(x for x in invList.host_entries() if x in hosts)
+        self.assertEqual(sorted(hosts), sorted(expected_hosts))
+        os.remove(hostfile)
+
+    def test_inventory_host_add_hosts_hostfile(self):
+        hosts = ("spam-spam", "eggs-spam", "foo-spam",)
+        hostf = ("spam-eggs-spam", "spam-foo-spam",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hostf:
+                f.write("{}\n".format(x))
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="add", hosts=hosts,
+                                          hostfile=hostfile)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts + hostf)
+        expected_hosts = tuple(invList.host_entries())
+        self.assertEqual(sorted(hosts + hostf), sorted(expected_hosts))
+        os.remove(hostfile)
+
+    def test_inventory_host_del_hosts(self):
+        hosts = ("web1", "shell1",)
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="del", hosts=hosts)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts)
+        expected_hosts = tuple(invList.host_entries())
+        self.assertTupleEqual(expected_hosts, ())
+
+    def test_inventory_host_del_hostfile(self):
+        hosts = ("loadbalancer3", "loadbalancer4",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hosts:
+                f.write("{}\n".format(x))
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="del", hostfile=hostfile)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts)
+        expected_hosts = tuple(invList.host_entries())
+        self.assertTupleEqual(expected_hosts, ())
+        os.remove(hostfile)
+
+    def test_inventory_host_del_hosts_hostfile(self):
+        hosts = ("loadbalancer1", "loadbalancer2",)
+        hostf = ("web2", "shell2",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hostf:
+                f.write("{}\n".format(x))
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="del", hosts=hosts,
+                                          hostfile=hostfile)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts + hostf)
+        expected_hosts = tuple(invList.host_entries())
+        self.assertTupleEqual(expected_hosts, ())
+        os.remove(hostfile)
+
+    @unittest.expectedFailure
+    def test_inventory_host_invalid_host(self):
+        try:
+            invalid_hostfile = op.join(inventory_dir, "invalid")
+            os.mkdir(invalid_hostfile)
+            hosts = ("invalid",)
+            invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                              action="del", hosts=hosts)
+            invHost.run()
+        except e:
+            os.rmdir(invalid_hostfile)
+            raise e
+
+    # InventoryTag
+    def test_inventory_tag_init(self):
+        invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                        action="add")
+        self.assertTrue(invTag.allhosts)
+        self.assertEqual(invTag.tagfile, "-")
+
+    def test_inventory_tag_stdin_multiple_hosts(self):
+        try:
+            invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                            action="add", tagfile="-",
+                                            hosts=("host1", "host2",))
+        except e:
+            self.fail()
+
+    def test_inventory_tag_stdin_hostfile(self):
+        try:
+            invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                            action="add", tagfile="-",
+                                            hostfile="hosts")
+        except e:
+            self.fail()
+
+    @unittest.expectedFailure
+    def test_inventory_tag_stdin_both(self):
+        invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                        action="add", tagfile="-",
+                                        hostfile="-")
+
+    def test_inventory_tag_add_for_all_hosts(self):
+        tags = ("spam-spam-spam", "spam-spam-eggs",)
+        tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",)
+        tagfile = op.join(fixtures, "tags")
+        with open(tagfile, "w") as f:
+            for x in tagsf:
+                f.write("{}\n".format(x))
+        invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                        action="add", tags=tags,
+                                        tagfile=tagfile)
+        invTag.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir)
+        failed = False
+        for host, taglist in invList.entries():
+            for x in tagsf + tags:
+                if x not in taglist:
+                    failed = True
+                    break
+            if failed:
+                break
+        os.remove(tagfile)
+        if failed:
+            self.fail()
+
+    def test_inventory_tag_add(self):
+        tags = ("spam-spam-spam", "spam-spam-eggs",)
+        tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",)
+        hosts = ("loadbalancer1", "loadbalancer2", "shell2",)
+        hostsf = ("web2", "web3",)
+        tagfile = op.join(fixtures, "tags")
+        with open(tagfile, "w") as f:
+            for x in tagsf:
+                f.write("{}\n".format(x))
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hostsf:
+                f.write("{}\n".format(x))
+        invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                        action="add", tags=tags,
+                                        tagfile=tagfile, hosts=hosts,
+                                        hostfile=hostfile)
+        invTag.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts + hostsf)
+        failed = False
+        for host, taglist in invList.entries():
+            if host not in hosts + hostsf:
+                failed = True
+                break
+            for x in tagsf + tags:
+                if x not in taglist:
+                    failed = True
+                    break
+            if failed:
+                break
+        os.remove(tagfile)
+        os.remove(hostfile)
+        if failed:
+            self.fail()
+
+    def test_inventory_tag_del_for_all_hosts(self):
+        tags = ("all",)
+        tagsf = ("charge",)
+        tagfile = op.join(fixtures, "tags")
+        with open(tagfile, "w") as f:
+            for x in tagsf:
+                f.write("{}\n".format(x))
+        invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                        action="del", tags=tags,
+                                        tagfile=tagfile)
+        invTag.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir)
+        failed = False
+        for host, taglist in invList.entries():
+            for x in tagsf + tags:
+                if x in taglist:
+                    failed = True
+                    break
+            if failed:
+                break
+        os.remove(tagfile)
+        if failed:
+            self.fail()
+
+    def test_inventory_tag_del(self):
+        tags = ("europe", "africa",)
+        tagsf = ("free", )
+        hosts = ("loadbalancer1", "loadbalancer2", "shell2",)
+        hostsf = ("web2", "web3",)
+        tagfile = op.join(fixtures, "tags")
+        with open(tagfile, "w") as f:
+            for x in tagsf:
+                f.write("{}\n".format(x))
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hostsf:
+                f.write("{}\n".format(x))
+        invTag = inventory.InventoryTag(db_basedir=inventory_dir,
+                                        action="del", tags=tags,
+                                        tagfile=tagfile, hosts=hosts,
+                                        hostfile=hostfile)
+        invTag.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts + hostsf)
+        failed = False
+        for host, taglist in invList.entries():
+            if host not in hosts + hostsf:
+                failed = True
+                break
+            for x in tagsf + tags:
+                if x in taglist:
+                    failed = True
+                    break
+            if failed:
+                break
+        os.remove(tagfile)
+        os.remove(hostfile)
+        if failed:
+            self.fail()
+
+    def test_inventory_tag_del_all_tags(self):
+        hosts = ("web3", "shell1",)
+        hostsf = ("shell2", "loadbalancer1",)
+        hostfile = op.join(fixtures, "hosts")
+        with open(hostfile, "w") as f:
+            for x in hostsf:
+                f.write("{}\n".format(x))
+        invHost = inventory.InventoryHost(db_basedir=inventory_dir,
+                                          action="del", all=True,
+                                          hosts=hosts, hostfile=hostfile)
+        invHost.run()
+        invList = inventory.InventoryList(db_basedir=inventory_dir,
+                                          hosts=hosts + hostsf)
+        for host, htags in invList.entries():
+            self.assertEqual(htags, ())
+        os.remove(hostfile)
+
+if __name__ == "__main__":
+    unittest.main()

cdist/test/manifest/__init__.py

@@ -53,6 +53,7 @@ class ManifestTestCase(test.CdistTestCase):
         base_root_path = os.path.join(out_path, hostdir)
         self.local = local.Local(
             target_host=self.target_host,
+            target_host_tags=self.target_host_tags,
             base_root_path=base_root_path,
             host_dir_name=hostdir,
             exec_path=cdist.test.cdist_exec_path,
@@ -93,6 +94,8 @@ class ManifestTestCase(test.CdistTestCase):
                          self.local.type_path)
         self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
         self.assertEqual(output_dict['__files'], self.local.files_path)
+        self.assertEqual(output_dict['__target_host_tags'],
+                         self.local.target_host_tags)
 
     def test_type_manifest_environment(self):
         cdist_type = core.CdistType(self.local.type_path, '__dump_environment')
@@ -126,6 +129,8 @@ class ManifestTestCase(test.CdistTestCase):
         self.assertEqual(output_dict['__object_id'], cdist_object.object_id)
         self.assertEqual(output_dict['__object_name'], cdist_object.name)
         self.assertEqual(output_dict['__files'], self.local.files_path)
+        self.assertEqual(output_dict['__target_host_tags'],
+                         self.local.target_host_tags)
 
     def test_debug_env_setup(self):
         current_level = self.log.getEffectiveLevel()

cdist/test/manifest/fixtures/conf/manifest/dump_environment

@@ -9,4 +9,5 @@ __global: $__global
 __cdist_type_base_path: $__cdist_type_base_path
 __manifest: $__manifest
 __files: $__files
+__target_host_tags: $__target_host_tags
 DONE

cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest

@@ -13,4 +13,5 @@ __object: $__object
 __object_id: $__object_id
 __object_name: $__object_name
 __files: $__files
+__target_host_tags: $__target_host_tags
 DONE

completions/bash/cdist-completion.bash

@@ -5,8 +5,8 @@ _cdist()
     cur="${COMP_WORDS[COMP_CWORD]}"
     prev="${COMP_WORDS[COMP_CWORD-1]}"
     prevprev="${COMP_WORDS[COMP_CWORD-2]}"
-    opts="-h --help -d --debug -v --verbose -V --version"
+    opts="-h --help -q --quiet -v --verbose -V --version"
-    cmds="banner shell config install"
+    cmds="banner config install inventory shell"
 
     case "${prevprev}" in
         shell)
@@ -18,6 +18,41 @@ _cdist()
                     ;;
             esac
             ;;
+         inventory)
+            case "${prev}" in
+                list)
+                    opts="-h --help -q --quiet -v --verbose -b --beta \
+                        -I --invento/y -a --all -f --file -H --host-only \
+                        -t --tag"
+                    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                    return 0
+                    ;;
+                add-host)
+                    opts="-h --help -q --quiet -v --verbose -b --beta \
+                        -I --inventory -f --file"
+                    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                    return 0
+                    ;;
+                del-host)
+                    opts="-h --help -q --quiet -v --verbose -b --beta \
+                        -I --inventory -a --all -f --file"
+                    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                    return 0
+                    ;;
+                add-tag)
+                    opts="-h --help -q --quiet -v --verbose -b --beta \
+                        -I --inventory -f --file -T --tag-file -t --taglist"
+                    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                    return 0
+                    ;;
+                del-tag)
+                    opts="-h --help -q --quiet -v --verbose -b --beta \
+                        -I --inventory -a --all -f --file -T --tag-file -t --taglist"
+                    COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
+                    return 0
+                    ;;
+            esac
+            ;;
     esac
 
     case "${prev}" in
@@ -26,23 +61,31 @@ _cdist()
             return 0
             ;;
         banner)
-            opts="-h --help -d --debug -v --verbose"
+            opts="-h --help -q --quiet -v --verbose"
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
             return 0
             ;;
         shell)
-            opts="-h --help -d --debug -v --verbose -s --shell"
+            opts="-h --help -q --quiet -v --verbose -s --shell"
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
             return 0
             ;;
         config|install)
-            opts="-h --help -d --debug -v --verbose -b --beta \
+            opts="-h --help -q --quiet -v --verbose -b --beta \
-                -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \
+                -I --inventory -C --cache-path-pattern -c --conf-dir \
-                -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential \
+                -f --file -i --initial-manifest -A --all-tagged \
-                --remote-copy --remote-exec"
+                -j --jobs -n --dry-run -o --out-dir -p --parallel \
+                -r --remote-out-dir \
+                -s --sequential --remote-copy --remote-exec -t --tag -a --all"
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
             return 0
             ;;
+        inventory)
+            cmds="list add-host del-host add-tag del-tag"
+            opts="-h --help -q --quiet -v --verbose"
+            COMPREPLY=( $(compgen -W "${opts} ${cmds}" -- ${cur}) )
+            return 0
+            ;;
         *)
             ;;
     esac

completions/zsh/_cdist

@@ -11,16 +11,16 @@ _cdist()
 
     case $state in
         opts_cmds)
-            _arguments '1:Options and commands:(banner config shell install -h --help -d --debug -v --verbose -V --version)'
+            _arguments '1:Options and commands:(banner config install inventory shell -h --help -q --quiet -v --verbose -V --version)'
             ;;
         *)
             case $words[2] in
                 -*)
-                    opts=(-h --help -d --debug -v --verbose -V --version)
+                    opts=(-h --help -q --quiet -v --verbose -V --version)
                     compadd "$@" -- $opts
                     ;;
                 banner)
-                    opts=(-h --help -d --debug -v --verbose)
+                    opts=(-h --help -q --quiet -v --verbose)
                     compadd "$@" -- $opts
                     ;;
                 shell)
@@ -30,16 +30,45 @@ _cdist()
                             compadd "$@" -- $shells
                             ;;
                         *)
-                            opts=(-h --help -d --debug -v --verbose -s --shell)
+                            opts=(-h --help -q --quiet -v --verbose -s --shell)
                             compadd "$@" -- $opts
                             ;;
                     esac
                     ;;
                 config|install)
-                    opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec)
+                    opts=(-h --help -q --quiet -v --verbose -a --all -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec -t --tag -I --inventory -A --all-tagged)
                     compadd "$@" -- $opts
                     ;;
-               *)
+                inventory)
+                    case $words[3] in
+                        list)
+                            opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -H --host-only -t --tag)
+                            compadd "$@" -- $opts
+                            ;;
+                        add-host)
+                            opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file)
+                            compadd "$@" -- $opts
+                            ;;
+                        del-host)
+                            opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file)
+                            compadd "$@" -- $opts
+                            ;;
+                        add-tag)
+                            opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file -T --tag-file -t --taglist)
+                            compadd "$@" -- $opts
+                            ;;
+                        del-tag)
+                            opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -T --tag-file -t --taglist)
+                            compadd "$@" -- $opts
+                            ;;
+                        *)
+                            cmds=(list add-host del-host add-tag del-tag)
+                            opts=(-h --help -q --quiet -v --verbose)
+                            compadd "$@" -- $cmds $opts
+                            ;;
+                    esac
+                    ;;
+                *)
                     ;;
             esac
     esac

docs/changelog

@@ -2,6 +2,11 @@ Changelog
 ---------
 
 next:
+	* Core: Add inventory functionality (Darko Poljak)
+	* Core: Expose inventory host tags in __target_host_tags env var (Darko Poljak)
+	* Type __timezone: Check current timezone before doing anything (Ander Punnar)
+
+4.5.0: 2017-07-20
 	* Types: Fix install types (Steven Armstrong)
 	* Core: Add -r command line option for setting remote base path (Steven Armstrong)
 	* Core: Allow manifest and gencode scripts to be written in any language (Darko Poljak)

docs/src/cdist-best-practice.rst

@@ -97,7 +97,7 @@ Including a possible common base that is reused across the different sites::
     git merge common
 
 
-The following **.git/config** is taken from a real world scenario:
+The following **.git/config** is taken from a real world scenario::
 
     # Track upstream, merge from time to time
     [remote "upstream"]

docs/src/cdist-inventory.rst

@@ -0,0 +1,211 @@
+Inventory
+=========
+
+Introduction
+------------
+
+cdist comes with simple built-in tag based inventory. It is a simple inventory
+with list of hosts and a host has a list of tags.
+Inventory functionality is still in **beta** so it can be used only if beta
+command line flag is specified (-b, --beta) or setting CDIST_BETA env var.
+
+Description
+-----------
+
+The idea is to have simple tagging inventory. There is a list of hosts and for
+each host there are tags. Inventory database is a set of files under inventory
+database base directory. Filename equals hostname. Each file contains tags for
+hostname with each tag on its own line.
+
+Using inventory you can now configure hosts by selecting them by tags.
+
+Tags have no values, as tags are just tags. Tag name-value would in this
+context mean that host has two tags and it is selected by specifying that both
+tags are present.
+
+This inventory is **KISS** cdist built-in inventory database. You can maintain it
+using cdist inventory interface or using standard UNIX tools.
+
+cdist inventory interface
+-------------------------
+
+With cdist inventory interface you can list host(s) and tag(s), add host(s),
+add tag(s), delete host(s) and delete tag(s).
+
+Configuring hosts using inventory
+---------------------------------
+
+config command now has new options, **-t**, **-a** and **-A**.
+
+**-A** means that all hosts in tag db is selected.
+
+**-a** means that selected hosts must contain ALL specified tags.
+
+**-t** means that host specifies tag - all hosts that have specified tags are
+selected.
+
+Examples
+--------
+
+.. code-block:: sh
+
+    # List inventory content
+    $ cdist inventory list -b
+
+    # List inventory for specified host localhost
+    $ cdist inventory list -b localhost
+
+    # List inventory for specified tag loadbalancer
+    $ cdist inventory list -b -t loadbalancer
+
+    # Add hosts to inventory
+    $ cdist inventory add-host -b web1 web2 web3
+
+    # Delete hosts from file old-hosts from inventory
+    $ cdist inventory del-host -b -f old-hosts
+
+    # Add tags to specifed hosts
+    $ cdist inventory add-tag -b -t europe,croatia,web,static web1 web2
+
+    # Add tag to all hosts in inventory
+    $ cdist inventory add-tag -b -t vm
+
+    # Delete all tags from specified host
+    $ cdist inventory del-tag -b -a localhost
+
+    # Delete tags read from stdin from hosts specified by file hosts
+    $ cdist inventory del-tag -b -T - -f hosts
+
+    # Configure hosts from inventory with any of specified tags
+    $ cdist config -b -t web dynamic
+
+    # Configure hosts from inventory with all specified tags
+    $ cdist config -b -t -a web dynamic
+
+    # Configure all hosts from inventory db
+    $ cdist config -b -A
+
+Example of manipulating database
+--------------------------------
+
+.. code-block:: sh
+
+    $ python3 scripts/cdist inventory list -b
+    $ python3 scripts/cdist inventory add-host -b localhost
+    $ python3 scripts/cdist inventory add-host -b test.mycloud.net
+    $ python3 scripts/cdist inventory list -b
+    localhost
+    test.mycloud.net
+    $ python3 scripts/cdist inventory add-host -b web1.mycloud.net web2.mycloud.net shell1.mycloud.net shell2.mycloud.net
+    $ python3 scripts/cdist inventory list -b
+    localhost
+    test.mycloud.net
+    web1.mycloud.net
+    web2.mycloud.net
+    shell1.mycloud.net
+    shell2.mycloud.net
+    $ python3 scripts/cdist inventory add-tag -b -t web web1.mycloud.net web2.mycloud.net
+    $ python3 scripts/cdist inventory add-tag -b -t shell shell1.mycloud.net shell2.mycloud.net
+    $ python3 scripts/cdist inventory add-tag -b -t cloud
+    $ python3 scripts/cdist inventory list -b
+    localhost cloud
+    test.mycloud.net cloud
+    web1.mycloud.net cloud,web
+    web2.mycloud.net cloud,web
+    shell1.mycloud.net cloud,shell
+    shell2.mycloud.net cloud,shell
+    $ python3 scripts/cdist inventory add-tag -b -t test,web,shell test.mycloud.net
+    $ python3 scripts/cdist inventory list -b
+    localhost cloud
+    test.mycloud.net cloud,shell,test,web
+    web1.mycloud.net cloud,web
+    web2.mycloud.net cloud,web
+    shell1.mycloud.net cloud,shell
+    shell2.mycloud.net cloud,shell
+    $ python3 scripts/cdist inventory del-tag -b -t shell test.mycloud.net
+    $ python3 scripts/cdist inventory list -b
+    localhost cloud
+    test.mycloud.net cloud,test,web
+    web1.mycloud.net cloud,web
+    web2.mycloud.net cloud,web
+    shell1.mycloud.net cloud,shell
+    shell2.mycloud.net cloud,shell
+    $ python3 scripts/cdist inventory add-tag -b -t all
+    $ python3 scripts/cdist inventory add-tag -b -t mistake
+    $ python3 scripts/cdist inventory list -b
+    localhost all,cloud,mistake
+    test.mycloud.net all,cloud,mistake,test,web
+    web1.mycloud.net all,cloud,mistake,web
+    web2.mycloud.net all,cloud,mistake,web
+    shell1.mycloud.net all,cloud,mistake,shell
+    shell2.mycloud.net all,cloud,mistake,shell
+    $ python3 scripts/cdist inventory del-tag -b -t mistake
+    $ python3 scripts/cdist inventory list -b
+    localhost all,cloud
+    test.mycloud.net all,cloud,test,web
+    web1.mycloud.net all,cloud,web
+    web2.mycloud.net all,cloud,web
+    shell1.mycloud.net all,cloud,shell
+    shell2.mycloud.net all,cloud,shell
+    $ python3 scripts/cdist inventory del-host -b localhost
+    $ python3 scripts/cdist inventory list -b
+    test.mycloud.net all,cloud,test,web
+    web1.mycloud.net all,cloud,web
+    web2.mycloud.net all,cloud,web
+    shell1.mycloud.net all,cloud,shell
+    shell2.mycloud.net all,cloud,shell
+    $ python3 scripts/cdist inventory list -b -t web
+    test.mycloud.net all,cloud,test,web
+    web1.mycloud.net all,cloud,web
+    web2.mycloud.net all,cloud,web
+    $ python3 scripts/cdist inventory list -b -t -a web test
+    test.mycloud.net all,cloud,test,web
+    $ python3 scripts/cdist inventory list -b -t -a web all
+    test.mycloud.net all,cloud,test,web
+    web1.mycloud.net all,cloud,web
+    web2.mycloud.net all,cloud,web
+    $ python3 scripts/cdist inventory list -b -t web all
+    test.mycloud.net all,cloud,test,web
+    web1.mycloud.net all,cloud,web
+    web2.mycloud.net all,cloud,web
+    shell1.mycloud.net all,cloud,shell
+    shell2.mycloud.net all,cloud,shell
+    $ cd cdist/inventory
+    $ ls -1
+    shell1.mycloud.net
+    shell2.mycloud.net
+    test.mycloud.net
+    web1.mycloud.net
+    web2.mycloud.net
+    $ ls -l
+    total 20
+    -rw-r--r--  1 darko  darko  16 Jun 24 12:43 shell1.mycloud.net
+    -rw-r--r--  1 darko  darko  16 Jun 24 12:43 shell2.mycloud.net
+    -rw-r--r--  1 darko  darko  19 Jun 24 12:43 test.mycloud.net
+    -rw-r--r--  1 darko  darko  14 Jun 24 12:43 web1.mycloud.net
+    -rw-r--r--  1 darko  darko  14 Jun 24 12:43 web2.mycloud.net
+    $ cat test.mycloud.net
+    test
+    all
+    web
+    cloud
+    $ cat web2.mycloud.net
+    all
+    web
+    cloud
+
+For more info about inventory commands and options see `cdist <man1/cdist.html>`_\ (1).
+
+Using external inventory
+------------------------
+
+cdist can be used with any external inventory where external inventory is
+some storage or database from which you can get a list of hosts to configure.
+cdist can then be fed with this list of hosts through stdin or file using
+**-f** option. For example, if your host list is stored in sqlite3 database
+hosts.db and you want to select hosts which purpose is **django** then you
+can use it with cdist like:
+
+.. code-block:: sh
+
+    $ sqlite3 hosts.db "select hostname from hosts where purpose = 'django';" | cdist config

docs/src/cdist-reference.rst.sh

@@ -63,6 +63,10 @@ cdist/conf/
     The distribution configuration directory.
     This contains types and explorers to be used.
 
+cdist/inventory/
+    The distribution inventory directory.
+    This path is relative to cdist installation directory.
+
 confdir
     Cdist will use all available configuration directories and create
     a temporary confdir containing links to the real configuration directories.
@@ -239,6 +243,9 @@ __target_fqdn
     This variable is derived from **__target_host**
     (using **socket.getfqdn()**).
     Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
+__target_host_tags
+    Comma separated list of target host tags.
+    Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell.
 __type
     Path to the current type.
     Available for: type manifest, type gencode.
@@ -274,6 +281,9 @@ CDIST_REMOTE_EXEC
 CDIST_REMOTE_COPY
     Use this command for remote copy (should behave like scp).
 
+CDIST_INVENTORY_DIR
+    Use this directory as inventory directory.
+
 CDIST_BETA
     Enable beta functionalities.
 

docs/src/cdist-type.rst

@@ -64,15 +64,23 @@ If a type is flagged with 'install' flag then it is used only with install comma
 With other commands, i.e. config, these types are skipped if used.
 
 
+Nonparallel types
+-----------------
+If a type is flagged with 'nonparallel' flag then its objects cannot be run in parallel
+when using -j option. Example of such a type is __package_dpkg type where dpkg itself
+prevents to be run in more than one instance.
+
+
 How to write a new type
 -----------------------
 A type consists of
 
-- parameter (optional)
+- parameter    (optional)
-- manifest  (optional)
+- manifest     (optional)
-- singleton (optional)
+- singleton    (optional)
-- explorer  (optional)
+- explorer     (optional)
-- gencode   (optional)
+- gencode      (optional)
+- nonparallel  (optional)
 
 Types are stored below cdist/conf/type/. Their name should always be prefixed with
 two underscores (__) to prevent collisions with other executables in $PATH.
@@ -240,6 +248,19 @@ install: create the (empty) file "install" in your type directory:
 With other commands, i.e. config, it will be skipped if used.
 
 
+Nonparallel - only one instance can be run at a time
+----------------------------------------------------
+If objects of a type must not or cannot be run in parallel when using -j
+option, you must mark it as nonparallel: create the (empty) file "nonparallel"
+in your type directory:
+
+.. code-block:: sh
+
+    touch cdist/conf/type/__NAME/nonparallel
+
+For example, package types are nonparallel types.
+
+
 The type explorers
 ------------------
 If a type needs to explore specific details, it can provide type specific

docs/src/index.rst

@@ -24,6 +24,7 @@ Contents:
    cdist-explorer
    cdist-messaging
    cdist-parallelization
+   cdist-inventory
    cdist-reference
    cdist-best-practice
    cdist-stages

docs/src/man1/cdist.rst

@@ -11,23 +11,48 @@ SYNOPSIS
 
 ::
 
-    cdist [-h] [-v] [-V] {banner,config,shell,install} ...
+    cdist [-h] [-q] [-v] [-V] {banner,config,install,inventory,shell} ...
 
-    cdist banner [-h] [-v]
+    cdist banner [-h] [-q] [-v]
 
-    cdist config [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
+    cdist config [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
                  [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
-                 [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                 [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
-                 [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s]
+                 [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
-                 [host [host ...]]
+                 [-f HOSTFILE] [-p] [-s] [-t]
+                 [host [host ...]] 
 
-    cdist install [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
+    cdist install [-h] [-q] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
                   [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
-                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
+                  [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
-                  [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s]
+                  [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
-                  [host [host ...]]
+                  [-f HOSTFILE] [-p] [-s] [-t]
+                  [host [host ...]] 
 
-    cdist shell [-h] [-v] [-s SHELL]
+    cdist inventory [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+                    {add-host,add-tag,del-host,del-tag,list} ...
+
+    cdist inventory add-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+                             [-f HOSTFILE]
+                             [host [host ...]]
+
+    cdist inventory add-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR]
+                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [host [host ...]]
+
+    cdist inventory del-host [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+                             [-f HOSTFILE]
+                             [host [host ...]]
+
+    cdist inventory del-tag [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+                            [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST]
+                            [host [host ...]]
+
+    cdist inventory list [-h] [-q] [-v] [-b] [-I INVENTORY_DIR] [-a]
+                         [-f HOSTFILE] [-H] [-t]
+                         [host [host ...]]
+
+    cdist shell [-h] [-q] [-v] [-s SHELL]
 
 
 DESCRIPTION
@@ -72,6 +97,15 @@ CONFIG/INSTALL
 --------------
 Configure/install one or more hosts.
 
+.. option:: -A, --all-tagged
+
+    use all hosts present in tags db
+
+.. option:: -a, --all
+
+    list hosts that have all specified tags, if -t/--tag
+    is specified
+
 .. option:: -b, --beta
 
     Enable beta functionality.
@@ -103,6 +137,16 @@ Configure/install one or more hosts.
     read hosts from stdin. For the file format see
     :strong:`HOSTFILE FORMAT` below.
 
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+    Use specified custom inventory directory. Inventory
+    directory is set up by the following rules: if this
+    argument is set then specified directory is used, if
+    CDIST_INVENTORY_DIR env var is set then its value is
+    used, if HOME env var is set then ~/.cdit/inventory is
+    used, otherwise distribution inventory directory is
+    used.
+
 .. option:: -i MANIFEST, --initial-manifest MANIFEST
 
     Path to a cdist manifest or - to read from stdin
@@ -125,7 +169,7 @@ Configure/install one or more hosts.
 
     Operate on multiple hosts in parallel
 
-.. option:: -r, --remote-out-dir
+.. option:: -r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH
 
     Directory to save cdist output in on the target host
 
@@ -141,6 +185,10 @@ Configure/install one or more hosts.
 
     Command to use for remote execution (should behave like ssh)
 
+.. option:: -t, --tag
+
+    host is specified by tag, not hostname/address; list
+    all hosts that contain any of specified tags
 
 HOSTFILE FORMAT
 ~~~~~~~~~~~~~~~
@@ -174,6 +222,246 @@ Resulting path is used to specify cache path subdirectory under which
 current host cache data are saved.
 
 
+INVENTORY
+---------
+Manage inventory database.
+Currently in beta with all sub-commands.
+
+
+INVENTORY ADD-HOST
+------------------
+Add host(s) to inventory database.
+
+.. option:: host
+
+    host(s) to add
+
+.. option:: -b, --beta
+
+    Enable beta functionalities. Beta functionalities
+    include inventory command with all sub-commands and
+    all options; config sub-command options: -j/--jobs,
+    -t/--tag, -a/--all.
+
+    Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+    Read additional hosts to add from specified file or
+    from stdin if '-' (each host on separate line). If no
+    host or host file is specified then, by default, read
+    from stdin. Hostfile format is the same as config hostfile format.
+
+.. option:: -h, --help
+
+    show this help message and exit
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+    Use specified custom inventory directory. Inventory
+    directory is set up by the following rules: if this
+    argument is set then specified directory is used, if
+    CDIST_INVENTORY_DIR env var is set then its value is
+    used, if HOME env var is set then ~/.cdist/inventory is
+    used, otherwise distribution inventory directory is
+    used.
+
+
+INVENTORY ADD-TAG
+-----------------
+Add tag(s) to inventory database.
+
+.. option:: host
+
+    list of host(s) for which tags are added
+
+.. option:: -b, --beta
+
+    Enable beta functionalities. Beta functionalities
+    include inventory command with all sub-commands and
+    all options; config sub-command options: -j/--jobs,
+    -t/--tag, -a/--all.
+
+    Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+    Read additional hosts to add tags from specified file
+    or from stdin if '-' (each host on separate line). If
+    no host or host file is specified then, by default,
+    read from stdin. If no tags/tagfile nor hosts/hostfile
+    are specified then tags are read from stdin and are
+    added to all hosts. Hostfile format is the same as config hostfile format.
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+    Use specified custom inventory directory. Inventory
+    directory is set up by the following rules: if this
+    argument is set then specified directory is used, if
+    CDIST_INVENTORY_DIR env var is set then its value is
+    used, if HOME env var is set then ~/.cdist/inventory is
+    used, otherwise distribution inventory directory is
+    used.
+
+.. option:: -T TAGFILE, --tag-file TAGFILE
+
+    Read additional tags to add from specified file or
+    from stdin if '-' (each tag on separate line). If no
+    tag or tag file is specified then, by default, read
+    from stdin. If no tags/tagfile nor hosts/hostfile are
+    specified then tags are read from stdin and are added
+    to all hosts. Tagfile format is the same as config hostfile format.
+
+.. option:: -t TAGLIST, --taglist TAGLIST
+
+    Tag list to be added for specified host(s), comma
+    separated values
+
+
+INVENTORY DEL-HOST
+------------------
+Delete host(s) from inventory database.
+
+.. option:: host
+
+    host(s) to delete
+
+.. option:: -a, --all
+
+    Delete all hosts
+
+.. option:: -b, --beta
+
+    Enable beta functionalities. Beta functionalities
+    include inventory command with all sub-commands and
+    all options; config sub-command options: -j/--jobs,
+    -t/--tag, -a/--all.
+
+    Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+    Read additional hosts to delete from specified file or
+    from stdin if '-' (each host on separate line). If no
+    host or host file is specified then, by default, read
+    from stdin. Hostfile format is the same as config hostfile format.
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+    Use specified custom inventory directory. Inventory
+    directory is set up by the following rules: if this
+    argument is set then specified directory is used, if
+    CDIST_INVENTORY_DIR env var is set then its value is
+    used, if HOME env var is set then ~/.cdist/inventory is
+    used, otherwise distribution inventory directory is
+    used.
+
+
+INVENTORY DEL-TAG
+-----------------
+Delete tag(s) from inventory database.
+
+.. option:: host
+
+    list of host(s) for which tags are deleted
+
+.. option:: -a, --all
+
+    Delete all tags for specified host(s)
+
+.. option:: -b, --beta
+
+    Enable beta functionalities. Beta functionalities
+    include inventory command with all sub-commands and
+    all options; config sub-command options: -j/--jobs,
+    -t/--tag, -a/--all.
+
+    Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+    Read additional hosts to delete tags for from
+    specified file or from stdin if '-' (each host on
+    separate line). If no host or host file is specified
+    then, by default, read from stdin. If no tags/tagfile
+    nor hosts/hostfile are specified then tags are read
+    from stdin and are deleted from all hosts. Hostfile
+    format is the same as config hostfile format.
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+    Use specified custom inventory directory. Inventory
+    directory is set up by the following rules: if this
+    argument is set then specified directory is used, if
+    CDIST_INVENTORY_DIR env var is set then its value is
+    used, if HOME env var is set then ~/.cdist/inventory is
+    used, otherwise distribution inventory directory is
+    used.
+
+.. option:: -T TAGFILE, --tag-file TAGFILE
+
+    Read additional tags from specified file or from stdin
+    if '-' (each tag on separate line). If no tag or tag
+    file is specified then, by default, read from stdin.
+    If no tags/tagfile nor hosts/hostfile are specified
+    then tags are read from stdin and are added to all
+    hosts. Tagfile format is the same as config hostfile format.
+
+.. option:: -t TAGLIST, --taglist TAGLIST
+
+    Tag list to be deleted for specified host(s), comma
+    separated values
+
+
+INVENTORY LIST
+--------------
+List inventory database.
+
+.. option::  host
+
+    host(s) to list
+
+.. option:: -a, --all
+
+    list hosts that have all specified tags, if -t/--tag
+    is specified
+
+.. option:: -b, --beta
+
+    Enable beta functionalities. Beta functionalities
+    include inventory command with all sub-commands and
+    all options; config sub-command options: -j/--jobs,
+    -t/--tag, -a/--all.
+
+    Can also be enabled using CDIST_BETA env var.
+
+.. option:: -f HOSTFILE, --file HOSTFILE
+
+    Read additional hosts to list from specified file or
+    from stdin if '-' (each host on separate line). If no
+    host or host file is specified then, by default, list
+    all. Hostfile format is the same as config hostfile format.
+
+.. option:: -H, --host-only
+
+    Suppress tags listing
+
+.. option:: -I INVENTORY_DIR, --inventory INVENTORY_DIR
+
+    Use specified custom inventory directory. Inventory
+    directory is set up by the following rules: if this
+    argument is set then specified directory is used, if
+    CDIST_INVENTORY_DIR env var is set then its value is
+    used, if HOME env var is set then ~/.cdist/inventory is
+    used, otherwise distribution inventory directory is
+    used.
+
+.. option:: -t, --tag
+
+    host is specified by tag, not hostname/address; list
+    all hosts that contain any of specified tags
+
+
 SHELL
 -----
 This command allows you to spawn a shell that enables access
@@ -186,14 +474,21 @@ usage. Its primary use is for debugging type parameters.
     Select shell to use, defaults to current shell. Used shell should
     be POSIX compatible shell.
 
+
 FILES
 -----
 ~/.cdist
     Your personal cdist config directory. If exists it will be
     automatically used.
+~/.cdist/inventory
+    The home inventory directory. If ~/.cdist exists it will be used as
+    default inventory directory.
 cdist/conf
     The distribution configuration directory. It contains official types and
     explorers. This path is relative to cdist installation directory.
+cdist/inventory
+    The distribution inventory directory.
+    This path is relative to cdist installation directory.
 
 NOTES
 -----
@@ -243,6 +538,43 @@ EXAMPLES
     # Install ikq05.ethz.ch with debug enabled
     % cdist install -vvv ikq05.ethz.ch
 
+    # List inventory content
+    % cdist inventory list -b
+
+    # List inventory for specified host localhost
+    % cdist inventory list -b localhost
+
+    # List inventory for specified tag loadbalancer
+    % cdist inventory list -b -t loadbalancer
+
+    # Add hosts to inventory
+    % cdist inventory add-host -b web1 web2 web3
+
+    # Delete hosts from file old-hosts from inventory
+    % cdist inventory del-host -b -f old-hosts
+
+    # Add tags to specifed hosts
+    % cdist inventory add-tag -b -t europe,croatia,web,static web1 web2
+
+    # Add tag to all hosts in inventory
+    % cdist inventory add-tag -b -t vm
+
+    # Delete all tags from specified host
+    % cdist inventory del-tag -b -a localhost
+
+    # Delete tags read from stdin from hosts specified by file hosts
+    % cdist inventory del-tag -b -T - -f hosts
+
+    # Configure hosts from inventory with any of specified tags
+    % cdist config -b -t web dynamic
+
+    # Configure hosts from inventory with all specified tags
+    % cdist config -b -t -a web dynamic
+
+    # Configure all hosts from inventory db
+    $ cdist config -b -A
+
+
 ENVIRONMENT
 -----------
 TMPDIR, TEMP, TMP
@@ -272,6 +604,9 @@ CDIST_REMOTE_EXEC
 CDIST_REMOTE_COPY
     Use this command for remote copy (should behave like scp).
 
+CDIST_INVENTORY_DIR
+    Use this directory as inventory directory.
+
 CDIST_BETA
     Enable beta functionality.
 

scripts/cdist

@@ -33,6 +33,7 @@ def commandline():
     import cdist.config
     import cdist.install
     import cdist.shell
+    import cdist.inventory
     import shutil
     import os