Browse Source

Merge inventory from beta branch.

4.6
Darko Poljak 5 years ago
parent
commit
e2a1519332
  1. 3
      .gitignore
  2. 166
      cdist/argparse.py
  3. 6
      cdist/conf/type/__install_stage/man.rst
  4. 49
      cdist/config.py
  5. 3
      cdist/core/code.py
  6. 1
      cdist/core/explorer.py
  7. 2
      cdist/core/manifest.py
  8. 5
      cdist/exec/local.py
  9. 390
      cdist/inventory.py
  10. 3
      cdist/shell.py
  11. 1
      cdist/test/__init__.py
  12. 5
      cdist/test/code/__init__.py
  13. 1
      cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local
  14. 3
      cdist/test/config/__init__.py
  15. 7
      cdist/test/emulator/__init__.py
  16. 1
      cdist/test/explorer/__init__.py
  17. 476
      cdist/test/inventory/__init__.py
  18. 5
      cdist/test/manifest/__init__.py
  19. 1
      cdist/test/manifest/fixtures/conf/manifest/dump_environment
  20. 1
      cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest
  21. 59
      completions/bash/cdist-completion.bash
  22. 41
      completions/zsh/_cdist
  23. 4
      docs/changelog
  24. 211
      docs/src/cdist-inventory.rst
  25. 10
      docs/src/cdist-reference.rst.sh
  26. 1
      docs/src/index.rst
  27. 349
      docs/src/man1/cdist.rst
  28. 1
      scripts/cdist

3
.gitignore vendored

@ -11,6 +11,9 @@ docs/src/cdist-reference.rst
# Ignore cdist cache for version control
/cache/
# Ignore inventory basedir
cdist/inventory/
# Python: cache, distutils, distribution in general
__pycache__/
*.pyc

166
cdist/argparse.py vendored

@ -7,10 +7,10 @@ import collections
# set of beta sub-commands
BETA_COMMANDS = set(('install', ))
BETA_COMMANDS = set(('install', 'inventory', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
'config': set(('jobs', )),
'config': set(('jobs', 'tag', 'all_tagged_hosts', )),
}
EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
# Parser others can reuse
@ -121,6 +121,17 @@ def get_parsers():
'banner', parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
parser['inventory_common'] = argparse.ArgumentParser(add_help=False)
parser['inventory_common'].add_argument(
'-I', '--inventory',
help=('Use specified custom inventory directory. '
'Inventory directory is set up by the following rules: '
'if this argument is set then specified directory is used, '
'if CDIST_INVENTORY_DIR env var is set then its value is '
'used, if HOME env var is set then ~/.cdist/inventory is '
'used, otherwise distribution inventory directory is used.'),
dest="inventory_dir", required=False)
# Config
parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
@ -156,6 +167,10 @@ def get_parsers():
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config_main'].add_argument(
'-r', '--remote-out-dir',
help='Directory to save cdist output in on the target host',
dest="remote_out_path")
parser['config_main'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
@ -170,6 +185,15 @@ def get_parsers():
# Config
parser['config_args'] = argparse.ArgumentParser(add_help=False)
parser['config_args'].add_argument(
'-A', '--all-tagged',
help=('use all hosts present in tags db'),
action="store_true", dest="all_tagged_hosts", default=False)
parser['config_args'].add_argument(
'-a', '--all',
help=('list hosts that have all specified tags, '
'if -t/--tag is specified'),
action="store_true", dest="has_all_tags", default=False)
parser['config_args'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config_args'].add_argument(
@ -183,17 +207,19 @@ def get_parsers():
'-p', '--parallel',
help='operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config_args'].add_argument(
'-r', '--remote-out-dir',
help='Directory to save cdist output in on the target host',
dest="remote_out_path")
parser['config_args'].add_argument(
'-s', '--sequential',
help='operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
parser['config_args'].add_argument(
'-t', '--tag',
help=('host is specified by tag, not hostname/address; '
'list all hosts that contain any of specified tags'),
dest='tag', required=False, action="store_true", default=False)
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'],
parser['config_main'],
parser['inventory_common'],
parser['config_args']])
parser['config'].set_defaults(func=cdist.config.Config.commandline)
@ -202,6 +228,134 @@ def get_parsers():
parents=[parser['config']])
parser['install'].set_defaults(func=cdist.install.Install.commandline)
# Inventory
parser['inventory'] = parser['sub'].add_parser(
'inventory', parents=[parser['loglevel'], parser['beta'],
parser['inventory_common']])
parser['invsub'] = parser['inventory'].add_subparsers(
title="Inventory commands", dest="subcommand")
parser['add-host'] = parser['invsub'].add_parser(
'add-host', parents=[parser['loglevel'], parser['beta'],
parser['inventory_common']])
parser['add-host'].add_argument(
'host', nargs='*', help='host(s) to add')
parser['add-host'].add_argument(
'-f', '--file',
help=('Read additional hosts to add from specified file '
'or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'read from stdin.'),
dest='hostfile', required=False)
parser['add-tag'] = parser['invsub'].add_parser(
'add-tag', parents=[parser['loglevel'], parser['beta'],
parser['inventory_common']])
parser['add-tag'].add_argument(
'host', nargs='*',
help='list of host(s) for which tags are added')
parser['add-tag'].add_argument(
'-f', '--file',
help=('Read additional hosts to add tags from specified file '
'or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'read from stdin. If no tags/tagfile nor hosts/hostfile'
' are specified then tags are read from stdin and are'
' added to all hosts.'),
dest='hostfile', required=False)
parser['add-tag'].add_argument(
'-T', '--tag-file',
help=('Read additional tags to add from specified file '
'or from stdin if \'-\' (each tag on separate line). '
'If no tag or tag file is specified then, by default, '
'read from stdin. If no tags/tagfile nor hosts/hostfile'
' are specified then tags are read from stdin and are'
' added to all hosts.'),
dest='tagfile', required=False)
parser['add-tag'].add_argument(
'-t', '--taglist',
help=("Tag list to be added for specified host(s), comma separated"
" values"),
dest="taglist", required=False)
parser['del-host'] = parser['invsub'].add_parser(
'del-host', parents=[parser['loglevel'], parser['beta'],
parser['inventory_common']])
parser['del-host'].add_argument(
'host', nargs='*', help='host(s) to delete')
parser['del-host'].add_argument(
'-a', '--all', help=('Delete all hosts'),
dest='all', required=False, action="store_true", default=False)
parser['del-host'].add_argument(
'-f', '--file',
help=('Read additional hosts to delete from specified file '
'or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'read from stdin.'),
dest='hostfile', required=False)
parser['del-tag'] = parser['invsub'].add_parser(
'del-tag', parents=[parser['loglevel'], parser['beta'],
parser['inventory_common']])
parser['del-tag'].add_argument(
'host', nargs='*',
help='list of host(s) for which tags are deleted')
parser['del-tag'].add_argument(
'-a', '--all',
help=('Delete all tags for specified host(s)'),
dest='all', required=False, action="store_true", default=False)
parser['del-tag'].add_argument(
'-f', '--file',
help=('Read additional hosts to delete tags for from specified '
'file or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'read from stdin. If no tags/tagfile nor hosts/hostfile'
' are specified then tags are read from stdin and are'
' deleted from all hosts.'),
dest='hostfile', required=False)
parser['del-tag'].add_argument(
'-T', '--tag-file',
help=('Read additional tags from specified file '
'or from stdin if \'-\' (each tag on separate line). '
'If no tag or tag file is specified then, by default, '
'read from stdin. If no tags/tagfile nor'
' hosts/hostfile are specified then tags are read from'
' stdin and are added to all hosts.'),
dest='tagfile', required=False)
parser['del-tag'].add_argument(
'-t', '--taglist',
help=("Tag list to be deleted for specified host(s), "
"comma separated values"),
dest="taglist", required=False)
parser['list'] = parser['invsub'].add_parser(
'list', parents=[parser['loglevel'], parser['beta'],
parser['inventory_common']])
parser['list'].add_argument(
'host', nargs='*', help='host(s) to list')
parser['list'].add_argument(
'-a', '--all',
help=('list hosts that have all specified tags, '
'if -t/--tag is specified'),
action="store_true", dest="has_all_tags", default=False)
parser['list'].add_argument(
'-f', '--file',
help=('Read additional hosts to list from specified file '
'or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'list all.'), dest='hostfile', required=False)
parser['list'].add_argument(
'-H', '--host-only', help=('Suppress tags listing'),
action="store_true", dest="list_only_host", default=False)
parser['list'].add_argument(
'-t', '--tag',
help=('host is specified by tag, not hostname/address; '
'list all hosts that contain any of specified tags'),
action="store_true", default=False)
parser['inventory'].set_defaults(
func=cdist.inventory.Inventory.commandline)
# Shell
parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']])

6
cdist/conf/type/__install_stage/man.rst vendored

@ -17,9 +17,9 @@ REQUIRED PARAMETERS
uri
The uri from which to fetch the tarball.
Can be anything understood by curl, e.g:
| http://path/to/stage.tgz
| tftp:///path/to/stage.tgz
| file:///local/path/stage.tgz
| http://path/to/stage.tgz
| tftp:///path/to/stage.tgz
| file:///local/path/stage.tgz
OPTIONAL PARAMETERS

49
cdist/config.py vendored

@ -38,6 +38,9 @@ import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
from cdist import inventory
import cdist.util.ipaddr as ipaddr
from cdist import core
@ -134,11 +137,42 @@ class Config(object):
base_root_path = cls.create_base_root_path(args.out_path)
hostcnt = 0
for host in itertools.chain(cls.hosts(args.host),
cls.hosts(args.hostfile)):
if args.tag or args.all_tagged_hosts:
inventory.determine_default_inventory_dir(args)
if args.all_tagged_hosts:
inv_list = inventory.InventoryList(
hosts=None, istag=True, hostfile=None,
db_basedir=args.inventory_dir)
else:
inv_list = inventory.InventoryList(
hosts=args.host, istag=True, hostfile=args.hostfile,
db_basedir=args.inventory_dir,
has_all_tags=args.has_all_tags)
it = inv_list.entries()
else:
it = itertools.chain(cls.hosts(args.host),
cls.hosts(args.hostfile))
for entry in it:
if isinstance(entry, tuple):
# if configuring by specified tags
host = entry[0]
host_tags = entry[1]
else:
# if configuring by host then check inventory for tags
host = entry
inventory.determine_default_inventory_dir(args)
inv_list = inventory.InventoryList(
hosts=(host,), db_basedir=args.inventory_dir)
inv = tuple(inv_list.entries())
if inv:
# host is present in inventory and has tags
host_tags = inv[0][1]
else:
# host is not present in inventory or has no tags
host_tags = None
host_base_path, hostdir = cls.create_host_base_dirs(
host, base_root_path)
log.debug("Base root path for target host \"{}\" is \"{}\"".format(
host, host_base_path))
@ -147,11 +181,12 @@ class Config(object):
log.trace("Creating child process for %s", host)
process[host] = multiprocessing.Process(
target=cls.onehost,
args=(host, host_base_path, hostdir, args, True))
args=(host, host_tags, host_base_path, hostdir, args,
True))
process[host].start()
else:
try:
cls.onehost(host, host_base_path, hostdir,
cls.onehost(host, host_tags, host_base_path, hostdir,
args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
@ -199,7 +234,8 @@ class Config(object):
return (remote_exec, remote_copy, )
@classmethod
def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
parallel):
"""Configure ONE system"""
log = logging.getLogger(host)
@ -216,6 +252,7 @@ class Config(object):
local = cdist.exec.local.Local(
target_host=target_host,
target_host_tags=host_tags,
base_root_path=host_base_path,
host_dir_name=host_dir_name,
initial_manifest=args.manifest,

3
cdist/core/code.py vendored

@ -56,6 +56,7 @@ gencode-local
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type: full qualified path to the type's dir
__files: full qualified path to the files dir
__target_host_tags: comma spearated list of host tags
returns: string containing the generated code or None
@ -74,6 +75,7 @@ gencode-remote
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type: full qualified path to the type's dir
__files: full qualified path to the files dir
__target_host_tags: comma spearated list of host tags
returns: string containing the generated code or None
@ -106,6 +108,7 @@ class Code(object):
'__target_fqdn': self.target_host[2],
'__global': self.local.base_path,
'__files': self.local.files_path,
'__target_host_tags': self.local.target_host_tags,
}
def _run_gencode(self, cdist_object, which):

1
cdist/core/explorer.py vendored

@ -77,6 +77,7 @@ class Explorer(object):
'__target_hostname': self.target_host[1],
'__target_fqdn': self.target_host[2],
'__explorer': self.remote.global_explorer_path,
'__target_host_tags': self.local.target_host_tags,
}
self._type_explorers_transferred = []
self.jobs = jobs

2
cdist/core/manifest.py vendored

@ -42,6 +42,7 @@ common:
types are defined for use in type emulator
== local.type_path
__files: full qualified path to the files dir
__target_host_tags: comma spearated list of host tags
initial manifest is:
script: full qualified path to the initial manifest
@ -109,6 +110,7 @@ class Manifest(object):
'__target_hostname': self.target_host[1],
'__target_fqdn': self.target_host[2],
'__files': self.local.files_path,
'__target_host_tags': self.local.target_host_tags,
}
if self.log.getEffectiveLevel() == logging.DEBUG:

5
cdist/exec/local.py vendored

@ -49,6 +49,7 @@ class Local(object):
"""
def __init__(self,
target_host,
target_host_tags,
base_root_path,
host_dir_name,
exec_path=sys.argv[0],
@ -58,6 +59,10 @@ class Local(object):
quiet_mode=False):
self.target_host = target_host
if target_host_tags is None:
self.target_host_tags = ""
else:
self.target_host_tags = ",".join(target_host_tags)
self.hostdir = host_dir_name
self.base_path = os.path.join(base_root_path, "data")

390
cdist/inventory.py vendored

@ -0,0 +1,390 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import cdist
import logging
import os
import os.path
import itertools
import sys
from cdist.hostsource import hostfile_process_line
DIST_INVENTORY_DB_NAME = "inventory"
dist_inventory_db = os.path.abspath(os.path.join(
os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME))
def determine_default_inventory_dir(args):
# The order of inventory dir setting by decreasing priority
# 1. inventory_dir argument
# 2. CDIST_INVENTORY_DIR env var if set
# 3. ~/.cdist/inventory if HOME env var is set
# 4. distribution inventory directory
if not args.inventory_dir:
if 'CDIST_INVENTORY_DIR' in os.environ:
args.inventory_dir = os.environ['CDIST_INVENTORY_DIR']
else:
home = cdist.home_dir()
if home:
args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME)
else:
args.inventory_dir = dist_inventory_db
def contains_all(big, little):
"""Return True if big contains all elements from little,
False otherwise.
"""
return set(little).issubset(set(big))
def contains_any(big, little):
"""Return True if big contains any element from little,
False otherwise.
"""
for x in little:
if x in big:
return True
return False
def check_always_true(x, y):
return True
def rstrip_nl(s):
'''str.rstrip "\n" from s'''
return str.rstrip(s, "\n")
class Inventory(object):
"""Inventory main class"""
def __init__(self, db_basedir=dist_inventory_db):
self.db_basedir = db_basedir
self.log = logging.getLogger("inventory")
self.init_db()
def init_db(self):
self.log.debug("Init db: {}".format(self.db_basedir))
if not os.path.exists(self.db_basedir):
os.makedirs(self.db_basedir, exist_ok=True)
elif not os.path.isdir(self.db_basedir):
raise cdist.Error(("Invalid inventory db basedir \'{}\',"
" must be a directory").format(self.db_basedir))
@staticmethod
def strlist_to_list(slist):
if slist:
result = [x for x in slist.split(',') if x]
else:
result = []
return result
def _input_values(self, source):
"""Yield input values from source.
Source can be a sequence or filename (stdin if '-').
In case of filename each line represents one input value.
"""
if isinstance(source, str):
import fileinput
try:
with fileinput.FileInput(files=(source)) as f:
for x in f:
result = hostfile_process_line(x, strip_func=rstrip_nl)
if result:
yield result
except (IOError, OSError) as e:
raise cdist.Error("Error reading from \'{}\'".format(
source))
else:
if source:
for x in source:
if x:
yield x
def _host_path(self, host):
hostpath = os.path.join(self.db_basedir, host)
return hostpath
def _all_hosts(self):
return os.listdir(self.db_basedir)
def _check_host(self, hostpath):
if not os.path.exists(hostpath):
return False
else:
if not os.path.isfile(hostpath):
raise cdist.Error(("Host path \'{}\' exists, but is not"
" a valid file").format(hostpath))
return True
def _read_host_tags(self, hostpath):
result = set()
with open(hostpath, "rt") as f:
for tag in f:
tag = tag.rstrip("\n")
if tag:
result.add(tag)
return result
def _get_host_tags(self, host):
hostpath = self._host_path(host)
if self._check_host(hostpath):
return self._read_host_tags(hostpath)
else:
return None
def _write_host_tags(self, host, tags):
hostpath = self._host_path(host)
if self._check_host(hostpath):
with open(hostpath, "wt") as f:
for tag in tags:
f.write("{}\n".format(tag))
return True
else:
return False
@classmethod
def commandline(cls, args):
"""Manipulate inventory db"""
log = logging.getLogger("cdist")
if 'taglist' in args:
args.taglist = cls.strlist_to_list(args.taglist)
determine_default_inventory_dir(args)
log.info("Using inventory: {}".format(args.inventory_dir))
log.debug("Inventory args: {}".format(vars(args)))
log.debug("Inventory command: {}".format(args.subcommand))
if args.subcommand == "list":
c = InventoryList(hosts=args.host, istag=args.tag,
hostfile=args.hostfile,
db_basedir=args.inventory_dir,
list_only_host=args.list_only_host,
has_all_tags=args.has_all_tags)
elif args.subcommand == "add-host":
c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
db_basedir=args.inventory_dir)
elif args.subcommand == "del-host":
c = InventoryHost(hosts=args.host, hostfile=args.hostfile,
all=args.all, db_basedir=args.inventory_dir,
action="del")
elif args.subcommand == "add-tag":
c = InventoryTag(hosts=args.host, tags=args.taglist,
hostfile=args.hostfile, tagfile=args.tagfile,
db_basedir=args.inventory_dir)
elif args.subcommand == "del-tag":
c = InventoryTag(hosts=args.host, tags=args.taglist,
hostfile=args.hostfile, tagfile=args.tagfile,
all=args.all, db_basedir=args.inventory_dir,
action="del")
else:
raise cdist.Error("Unknown inventory command \'{}\'".format(
args.subcommand))
c.run()
class InventoryList(Inventory):
def __init__(self, hosts=None, istag=False, hostfile=None,
list_only_host=False, has_all_tags=False,
db_basedir=dist_inventory_db):
super().__init__(db_basedir)
self.hosts = hosts
self.istag = istag
self.hostfile = hostfile
self.list_only_host = list_only_host
self.has_all_tags = has_all_tags
def _print(self, host, tags):
if self.list_only_host:
print("{}".format(host))
else:
print("{} {}".format(host, ",".join(sorted(tags))))
def _do_list(self, it_tags, it_hosts, check_func):
if (it_tags is not None):
param_tags = set(it_tags)
self.log.debug("param_tags: {}".format(param_tags))
else:
param_tags = set()
for host in it_hosts:
self.log.debug("host: {}".format(host))
tags = self._get_host_tags(host)
if tags is None:
self.log.info("Host \'{}\' not found, skipped".format(host))
continue
self.log.debug("tags: {}".format(tags))
if check_func(tags, param_tags):
yield host, tags
def entries(self):
if not self.hosts and not self.hostfile:
self.log.info("Listing all hosts")
it_hosts = self._all_hosts()
it_tags = None
check_func = check_always_true
else:
it = itertools.chain(self._input_values(self.hosts),
self._input_values(self.hostfile))
if self.istag:
self.log.info("Listing by tag(s)")
it_hosts = self._all_hosts()
it_tags = it
if self.has_all_tags:
check_func = contains_all
else:
check_func = contains_any
else:
self.log.info("Listing by host(s)")
it_hosts = it
it_tags = None
check_func = check_always_true
for host, tags in self._do_list(it_tags, it_hosts, check_func):
yield host, tags
def host_entries(self):
for host, tags in self.entries():
yield host
def run(self):
for host, tags in self.entries():
self._print(host, tags)
class InventoryHost(Inventory):
def __init__(self, hosts=None, hostfile=None,
db_basedir=dist_inventory_db, all=False, action="add"):
super().__init__(db_basedir)
self.actions = ("add", "del")
if action not in self.actions:
raise cdist.Error("Invalid action \'{}\', valid actions are:"
" {}\n".format(action, self.actions.keys()))
self.action = action
self.hosts = hosts
self.hostfile = hostfile
self.all = all
if not self.hosts and not self.hostfile:
self.hostfile = "-"
def _new_hostpath(self, hostpath):
# create empty file
with open(hostpath, "w"):
pass
def _action(self, host):
if self.action == "add":
self.log.info("Adding host \'{}\'".format(host))
elif self.action == "del":
self.log.info("Deleting host \'{}\'".format(host))
hostpath = self._host_path(host)
self.log.debug("hostpath: {}".format(hostpath))
if self.action == "add" and not os.path.exists(hostpath):
self._new_hostpath(hostpath)
else:
if not os.path.isfile(hostpath):
raise cdist.Error(("Host path \'{}\' is"
" not a valid file").format(hostpath))
if self.action == "del":
os.remove(hostpath)
def run(self):
if self.action == "del" and self.all:
self.log.debug("Doing for all hosts")
it = self._all_hosts()
else:
self.log.debug("Doing for specified hosts")
it = itertools.chain(self._input_values(self.hosts),
self._input_values(self.hostfile))
for host in it:
self._action(host)
class InventoryTag(Inventory):
def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None,
db_basedir=dist_inventory_db, all=False, action="add"):
super().__init__(db_basedir)
self.actions = ("add", "del")
if action not in self.actions:
raise cdist.Error("Invalid action \'{}\', valid actions are:"
" {}\n".format(action, self.actions.keys()))
self.action = action
self.hosts = hosts
self.tags = tags
self.hostfile = hostfile
self.tagfile = tagfile
self.all = all
if not self.hosts and not self.hostfile:
self.allhosts = True
else:
self.allhosts = False
if not self.tags and not self.tagfile:
self.tagfile = "-"
if self.hostfile == "-" and self.tagfile == "-":
raise cdist.Error("Cannot read both, hosts and tags, from stdin")
def _read_input_tags(self):
self.input_tags = set()
for tag in itertools.chain(self._input_values(self.tags),
self._input_values(self.tagfile)):
self.input_tags.add(tag)
def _action(self, host):
host_tags = self._get_host_tags(host)
if host_tags is None:
print("Host \'{}\' does not exist, skipping".format(host),
file=sys.stderr)
return
self.log.debug("existing host_tags: {}".format(host_tags))
if self.action == "del" and self.all:
host_tags = set()
else:
for tag in self.input_tags:
if self.action == "add":
self.log.info("Adding tag \'{}\' for host \'{}\'".format(
tag, host))
host_tags.add(tag)
elif self.action == "del":
self.log.info("Deleting tag \'{}\' for host \'{}\'".format(
tag, host))
if tag in host_tags:
host_tags.remove(tag)
self.log.debug("new host tags: {}".format(host_tags))
if not self._write_host_tags(host, host_tags):
self.log.info("{} does not exist, skipped".format(host))
def run(self):
if self.allhosts:
self.log.debug("Doing for all hosts")
it = self._all_hosts()
else:
self.log.debug("Doing for specified hosts")
it = itertools.chain(self._input_values(self.hosts),
self._input_values(self.hostfile))
if not(self.action == "del" and self.all):
self._read_input_tags()
for host in it:
self._action(host)

3
cdist/shell.py vendored

@ -44,6 +44,7 @@ class Shell(object):
"cdist-shell-no-target-host",
"cdist-shell-no-target-host",
)
self.target_host_tags = ""
host_dir_name = cdist.str_hash(self.target_host[0])
base_root_path = tempfile.mkdtemp()
@ -51,6 +52,7 @@ class Shell(object):
self.local = cdist.exec.local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=host_dir_name)
@ -77,6 +79,7 @@ class Shell(object):
'__manifest': self.local.manifest_path,
'__explorer': self.local.global_explorer_path,
'__files': self.local.files_path,
'__target_host_tags': self.local.target_host_tags,
}
self.env.update(additional_env)

1
cdist/test/__init__.py vendored

@ -42,6 +42,7 @@ class CdistTestCase(unittest.TestCase):
'cdisttesthost',
'cdisttesthost',
)
target_host_tags = "tag1,tag2,tag3"
def mkdtemp(self, **kwargs):
return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)

5
cdist/test/code/__init__.py vendored

@ -46,6 +46,7 @@ class CodeTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=cdist.test.cdist_exec_path,
@ -97,6 +98,8 @@ class CodeTestCase(test.CdistTestCase):
self.cdist_object.object_id)
self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
self.assertEqual(output_dict['__files'], self.local.files_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
def test_run_gencode_remote_environment(self):
output_string = self.code.run_gencode_remote(self.cdist_object)
@ -120,6 +123,8 @@ class CodeTestCase(test.CdistTestCase):
self.cdist_object.object_id)
self.assertEqual(output_dict['__object_name'], self.cdist_object.name)
self.assertEqual(output_dict['__files'], self.local.files_path)
self.assertEqual(output_dict['__target_host_tags'],
self.local.target_host_tags)
def test_transfer_code_remote(self):
self.cdist_object.code_remote = self.code.run_gencode_remote(

1
cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local vendored

@ -9,3 +9,4 @@ echo "echo __object: $__object"
echo "echo __object_id: $__object_id"
echo "echo __object_name: $__object_name"
echo "echo __files: $__files"
echo "echo __target_host_tags: $__target_host_tags"

3
cdist/test/config/__init__.py vendored

@ -60,6 +60,7 @@ class ConfigRunTestCase(test.CdistTestCase):
os.makedirs(self.host_base_path)
self.local = cdist.exec.local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir)
@ -164,6 +165,7 @@ class ConfigRunTestCase(test.CdistTestCase):
"""Test if the dryrun option is working like expected"""
drylocal = cdist.exec.local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
# exec_path can not derivated from sys.argv in case of unittest
@ -181,6 +183,7 @@ class ConfigRunTestCase(test.CdistTestCase):
"""Test to show dependency resolver warning message."""
local = cdist.exec.local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=os.path.abspath(os.path.join(

7
cdist/test/emulator/__init__.py vendored

@ -53,6 +53,7 @@ class EmulatorTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@ -156,6 +157,7 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@ -246,6 +248,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@ -279,6 +282,7 @@ class OverrideTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@ -322,6 +326,7 @@ class ArgumentsTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@ -445,6 +450,7 @@ class StdinTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,
@ -511,6 +517,7 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=host_base_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,

1
cdist/test/explorer/__init__.py vendored

@ -50,6 +50,7 @@ class ExplorerClassTestCase(test.CdistTestCase):
self.local = local.Local(
target_host=self.target_host,
target_host_tags=self.target_host_tags,
base_root_path=base_root_path,
host_dir_name=hostdir,
exec_path=test.cdist_exec_path,

476
cdist/test/inventory/__init__.py vendored

@ -0,0 +1,476 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import cdist
import os.path as op
import unittest
import sys
from cdist import test
from cdist import inventory
from io import StringIO
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
inventory_dir = op.join(fixtures, "inventory")
class InventoryTestCase(test.CdistTestCase):
def _create_host_with_tags(self, host, tags):
os.makedirs(inventory_dir, exist_ok=True)
hostfile = op.join(inventory_dir, host)
with open(hostfile, "w") as f:
for x in tags:
f.write("{}\n".format(x))
def setUp(self):
self.maxDiff = None
self.db = {
"loadbalancer1": ["loadbalancer", "all", "europe", ],
"loadbalancer2": ["loadbalancer", "all", "europe", ],
"loadbalancer3": ["loadbalancer", "all", "africa", ],
"loadbalancer4": ["loadbalancer", "all", "africa", ],
"web1": ["web", "all", "static", ],
"web2": ["web", "all", "dynamic", ],
"web3": ["web", "all", "dynamic", ],
"shell1": ["shell", "all", "free", ],
"shell2": ["shell", "all", "free", ],
"shell3": ["shell", "all", "charge", ],
"shell4": ["shell", "all", "charge", ],
"monty": ["web", "python", "shell", ],
"python": ["web", "python", "shell", ],
}
for x in self.db:
self.db[x] = sorted(self.db[x])
for host in self.db:
self._create_host_with_tags(host, self.db[host])
self.sys_stdout = sys.stdout
out = StringIO()
sys.stdout = out
def _get_output(self):
sys.stdout.flush()
output = sys.stdout.getvalue().strip()
return output
def tearDown(self):
sys.stdout = self.sys_stdout
shutil.rmtree(inventory_dir)
def test_inventory_create_db(self):
dbdir = op.join(fixtures, "foo")
inv = inventory.Inventory(db_basedir=dbdir)
self.assertTrue(os.path.isdir(dbdir))
self.assertEqual(inv.db_basedir, dbdir)
shutil.rmtree(inv.db_basedir)
# InventoryList
def test_inventory_list_print(self):
invList = inventory.InventoryList(db_basedir=inventory_dir)
invList.run()
output = self._get_output()
self.assertTrue(' ' in output)
def test_inventory_list_print_host_only(self):
invList = inventory.InventoryList(db_basedir=inventory_dir,
list_only_host=True)
invList.run()
output = self._get_output()
self.assertFalse(' ' in output)
def test_inventory_list_all(self):
invList = inventory.InventoryList(db_basedir=inventory_dir)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
self.assertEqual(db, self.db)
def test_inventory_list_by_host_hosts(self):
hosts = ("web1", "web2", "web3",)
invList = inventory.InventoryList(db_basedir=inventory_dir,
hosts=hosts)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
expected_db = {host: sorted(self.db[host]) for host in hosts}
self.assertEqual(db, expected_db)
def test_inventory_list_by_host_hostfile(self):
hosts = ("web1", "web2", "web3",)
hostfile = op.join(fixtures, "hosts")
with open(hostfile, "w") as f:
for x in hosts:
f.write("{}\n".format(x))
invList = inventory.InventoryList(db_basedir=inventory_dir,
hostfile=hostfile)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
expected_db = {host: sorted(self.db[host]) for host in hosts}
self.assertEqual(db, expected_db)
os.remove(hostfile)
def test_inventory_list_by_host_hosts_hostfile(self):
hosts = ("shell1", "shell4",)
hostsf = ("web1", "web2", "web3",)
hostfile = op.join(fixtures, "hosts")
with open(hostfile, "w") as f:
for x in hostsf:
f.write("{}\n".format(x))
invList = inventory.InventoryList(db_basedir=inventory_dir,
hosts=hosts, hostfile=hostfile)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
import itertools
expected_db = {host: sorted(self.db[host]) for host in
itertools.chain(hostsf, hosts)}
self.assertEqual(db, expected_db)
os.remove(hostfile)
def _gen_expected_db_for_tags(self, tags):
db = {}
for host in self.db:
for tag in tags:
if tag in self.db[host]:
db[host] = self.db[host]
break
return db
def _gen_expected_db_for_has_all_tags(self, tags):
db = {}
for host in self.db:
if set(tags).issubset(set(self.db[host])):
db[host] = self.db[host]
return db
def test_inventory_list_by_tag_hosts(self):
tags = ("web", "shell",)
invList = inventory.InventoryList(db_basedir=inventory_dir,
istag=True, hosts=tags)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
expected_db = self._gen_expected_db_for_tags(tags)
self.assertEqual(db, expected_db)
def test_inventory_list_by_tag_hostfile(self):
tags = ("web", "shell",)
tagfile = op.join(fixtures, "tags")
with open(tagfile, "w") as f:
for x in tags:
f.write("{}\n".format(x))
invList = inventory.InventoryList(db_basedir=inventory_dir,
istag=True, hostfile=tagfile)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
expected_db = self._gen_expected_db_for_tags(tags)
self.assertEqual(db, expected_db)
os.remove(tagfile)
def test_inventory_list_by_tag_hosts_hostfile(self):
tags = ("web", "shell",)
tagsf = ("dynamic", "europe",)
tagfile = op.join(fixtures, "tags")
with open(tagfile, "w") as f:
for x in tagsf:
f.write("{}\n".format(x))
invList = inventory.InventoryList(db_basedir=inventory_dir,
istag=True, hosts=tags,
hostfile=tagfile)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
import itertools
expected_db = self._gen_expected_db_for_tags(tags + tagsf)
self.assertEqual(db, expected_db)
os.remove(tagfile)
def test_inventory_list_by_tag_has_all_tags(self):
tags = ("web", "python", "shell",)
invList = inventory.InventoryList(db_basedir=inventory_dir,
istag=True, hosts=tags,
has_all_tags=True)
entries = invList.entries()
db = {host: sorted(tags) for host, tags in entries}
expected_db = self._gen_expected_db_for_has_all_tags(tags)
self.assertEqual(db, expected_db)
# InventoryHost
def test_inventory_host_add_hosts(self):
hosts = ("spam", "eggs", "foo",)
invHost = inventory.InventoryHost(db_basedir=inventory_dir,
action="add", hosts=hosts)
invHost.run()
invList = inventory.InventoryList(db_basedir=inventory_dir)
expected_hosts = tuple(x for x in invList.host_entries() if x in hosts)
self.assertEqual(sorted(hosts), sorted(expected_hosts))
def test_inventory_host_add_hostfile(self):
hosts = ("spam-new", "eggs-new", "foo-new",)
hostfile = op.join(fixtures, "hosts")
with open(hostfile, "w") as f:
for x in hosts:
f.write("{}\n".format(x))
invHost = inventory.InventoryHost(db_basedir=inventory_dir,
action="add", hostfile=hostfile)
invHost.run()
invList = inventory.InventoryList(db_basedir=inventory_dir)
expected_hosts = tuple(x for x in invList.host_entries() if x in hosts)
self.assertEqual(sorted(hosts), sorted(expected_hosts))
os.remove(hostfile)
def test_inventory_host_add_hosts_hostfile(self):