From 0bb251bdda0de1dfe7c577ff5bc15055e5591ef3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 13 May 2016 22:20:55 +0200 Subject: [PATCH 1/5] Fix suffix description for __block type. --- cdist/conf/type/__block/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__block/man.text b/cdist/conf/type/__block/man.text index 2312d293..97f538fa 100644 --- a/cdist/conf/type/__block/man.text +++ b/cdist/conf/type/__block/man.text @@ -34,7 +34,7 @@ prefix:: Defaults to #cdist:__block/$__object_id suffix:: - the prefix to add after the text. + the suffix to add after the text. Defaults to #/cdist:__block/$__object_id state:: From 4fce4a631c8f2603cddd9f2065ec13a599fc3f21 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 22 May 2016 09:22:39 +0200 Subject: [PATCH 2/5] Add -f option for reading hosts from file or stdin. --- cdist/config.py | 32 ++++++++++++++++++++++++++++++-- docs/man/man1/cdist.text | 10 +++++++++- scripts/cdist | 25 +++++++++++++++++++------ 3 files changed, 58 insertions(+), 9 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index bba9bfcb..fd08d460 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -53,6 +53,21 @@ class Config(object): self.local.create_files_dirs() self.remote.create_files_dirs() + @staticmethod + def hosts(source): + """Yield hosts from source. + Source can be a sequence or filename (stdin if \'-\'). + In case of filename each line represents one host. + """ + if isinstance(source, str): + import fileinput + for host in fileinput.input(files=(source)): + yield host.strip() # remove leading and trailing whitespace + else: + for host in source: + yield host + + @classmethod def commandline(cls, args): """Configure remote system""" @@ -60,6 +75,12 @@ class Config(object): # FIXME: Refactor relict - remove later log = logging.getLogger("cdist") + + if args.manifest == '-' and args.hostfile == '-': + raise cdist.Error("Cannot read both, manifest and host file, from stdin") + # if no host source is specified then read hosts from stdin + if not (args.hostfile or args.host): + args.hostfile = '-' initial_manifest_tempfile = None if args.manifest == '-': @@ -79,8 +100,15 @@ class Config(object): process = {} failed_hosts = [] time_start = time.time() + + hostcnt = 0 + if args.host: + host_source = args.host + else: + host_source = args.hostfile - for host in args.host: + for host in cls.hosts(host_source): + hostcnt += 1 if args.parallel: log.debug("Creating child process for %s", host) process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True)) @@ -101,7 +129,7 @@ class Config(object): failed_hosts.append(host) time_end = time.time() - log.info("Total processing time for %s host(s): %s", len(args.host), + log.info("Total processing time for %s host(s): %s", hostcnt, (time_end - time_start)) if len(failed_hosts) > 0: diff --git a/docs/man/man1/cdist.text b/docs/man/man1/cdist.text index e29ae3ae..ed2ce805 100644 --- a/docs/man/man1/cdist.text +++ b/docs/man/man1/cdist.text @@ -14,7 +14,7 @@ cdist [-h] [-d] [-v] [-V] {banner,config,shell} ... cdist banner [-h] [-d] [-v] -cdist config [-h] [-d] [-V] [-c CONF_DIR] [-i MANIFEST] [-p] [-s] host [host ...] +cdist config [-h] [-d] [-V] [-c CONF_DIR] [-f HOSTFILE] [-i MANIFEST] [-p] [-s] [host [host ...]] cdist shell [-h] [-d] [-v] [-s SHELL] @@ -63,6 +63,11 @@ Configure one or more hosts --conf-dir argument have higher precedence over those set through the environment variable. +-f HOSTFILE, --file HOSTFILE:: + Read hosts to operate on from specified file or from stdin if '-' + (each host on separate line). If no host or host file is + specified then, by default, read hosts from stdin. + -i MANIFEST, --initial-manifest MANIFEST:: Path to a cdist manifest or - to read from stdin @@ -105,6 +110,9 @@ EXAMPLES --remote-copy /path/to/my/remote/copy \ -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch +# Configure hosts in parallel by reading hosts from file loadbalancers +% cdist config -f loadbalancers + # Display banner cdist banner diff --git a/scripts/cdist b/scripts/cdist index 8e22aacb..0cfd01d6 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -67,12 +67,14 @@ def commandline(): action='store_true', default=False) # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION, + parser['main'] = argparse.ArgumentParser(description='cdist ' + + cdist.VERSION, parents=[parser['loglevel']]) parser['main'].add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + cdist.VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") + parser['sub'] = parser['main'].add_subparsers(title="Commands", + dest="command") # Banner parser['banner'] = parser['sub'].add_parser('banner', @@ -82,11 +84,16 @@ def commandline(): # Config parser['config'] = parser['sub'].add_parser('config', parents=[parser['loglevel']]) - parser['config'].add_argument('host', nargs='+', + parser['config'].add_argument('host', nargs='*', help='one or more hosts to operate on') parser['config'].add_argument('-c', '--conf-dir', - help='Add configuration directory (can be repeated, last one wins)', - action='append') + help=('Add configuration directory (can be repeated, ' + 'last one wins)'), action='append') + parser['config'].add_argument('-f', '--file', + help=('Read hosts to operate on from specified file or from stdin ' + 'if \'-\' (each host on separate line). If no host or host ' + 'file is specified then, by default, read hosts from stdin.'), + dest='hostfile', required=False) parser['config'].add_argument('-i', '--initial-manifest', help='Path to a cdist manifest or \'-\' to read from stdin.', dest='manifest', required=False) @@ -108,7 +115,8 @@ def commandline(): action='store', dest='remote_copy', default=os.environ.get('CDIST_REMOTE_COPY')) parser['config'].add_argument('--remote-exec', - help='Command to use for remote execution (should behave like ssh)', + help=('Command to use for remote execution ' + '(should behave like ssh)'), action='store', dest='remote_exec', default=os.environ.get('CDIST_REMOTE_EXEC')) parser['config'].set_defaults(func=cdist.config.Config.commandline) @@ -147,6 +155,11 @@ def commandline(): if args_dict['remote_copy'] is None: args.remote_copy = cdist.REMOTE_COPY + mux_opts + if args.command == 'config': + if args.manifest == '-' and args.hostfile == '-': + print('cdist config: error: cannot read both, manifest and host file, from stdin') + sys.exit(1) + log.debug(args) log.info("version %s" % cdist.VERSION) From fa5175fee5e0cd1ed2aa7d8b3d63b39cc3977eec Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 22 May 2016 09:45:08 +0200 Subject: [PATCH 3/5] Allow both hosts sources: command line args and file. --- cdist/config.py | 26 +++++++++++++++----------- docs/man/man1/cdist.text | 7 ++++--- scripts/cdist | 7 ++++--- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index fd08d460..f5e62ce1 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -26,6 +26,7 @@ import shutil import sys import time import pprint +import itertools import cdist @@ -61,11 +62,17 @@ class Config(object): """ if isinstance(source, str): import fileinput - for host in fileinput.input(files=(source)): - yield host.strip() # remove leading and trailing whitespace + try: + for host in fileinput.input(files=(source)): + # remove leading and trailing whitespace + yield host.strip() + except (IOError, OSError) as e: + raise cdist.Error("Error reading hosts from \'{}\'".format( + source)) else: - for host in source: - yield host + if source: + for host in source: + yield host @classmethod @@ -77,7 +84,8 @@ class Config(object): log = logging.getLogger("cdist") if args.manifest == '-' and args.hostfile == '-': - raise cdist.Error("Cannot read both, manifest and host file, from stdin") + raise cdist.Error(("Cannot read both, manifest and host file, " + "from stdin")) # if no host source is specified then read hosts from stdin if not (args.hostfile or args.host): args.hostfile = '-' @@ -102,12 +110,8 @@ class Config(object): time_start = time.time() hostcnt = 0 - if args.host: - host_source = args.host - else: - host_source = args.hostfile - - for host in cls.hosts(host_source): + for host in itertools.chain(cls.hosts(args.host), + cls.hosts(args.hostfile)): hostcnt += 1 if args.parallel: log.debug("Creating child process for %s", host) diff --git a/docs/man/man1/cdist.text b/docs/man/man1/cdist.text index ed2ce805..4e45ae81 100644 --- a/docs/man/man1/cdist.text +++ b/docs/man/man1/cdist.text @@ -64,9 +64,10 @@ Configure one or more hosts environment variable. -f HOSTFILE, --file HOSTFILE:: - Read hosts to operate on from specified file or from stdin if '-' - (each host on separate line). If no host or host file is - specified then, by default, read hosts from stdin. + Read additional hosts to operate on from specified file + or from stdin if '-' (each host on separate line). + If no host or host file is specified then, by default, + read hosts from stdin. -i MANIFEST, --initial-manifest MANIFEST:: Path to a cdist manifest or - to read from stdin diff --git a/scripts/cdist b/scripts/cdist index 0cfd01d6..ccdb5232 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -90,9 +90,10 @@ def commandline(): help=('Add configuration directory (can be repeated, ' 'last one wins)'), action='append') parser['config'].add_argument('-f', '--file', - help=('Read hosts to operate on from specified file or from stdin ' - 'if \'-\' (each host on separate line). If no host or host ' - 'file is specified then, by default, read hosts from stdin.'), + help=('Read additional hosts to operate on from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read hosts from stdin.'), dest='hostfile', required=False) parser['config'].add_argument('-i', '--initial-manifest', help='Path to a cdist manifest or \'-\' to read from stdin.', From 69f3759a8956823bdf89bc0c37bbb31635ac0d39 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 23 May 2016 17:24:17 +0200 Subject: [PATCH 4/5] Update changelog for option -f. --- docs/changelog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 3a644b6f..e7dc26ed 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * Core: Add -f option to read additional hosts from file/stdin (Darko Poljak) + 4.0.0: 2016-05-04 * Core: Fix bug with parallel hosts operation when output path is specifed (Darko Poljak) * Type __package_pip: Add support for running as specified user (useful for pip in venv) (Darko Poljak) @@ -25,7 +28,6 @@ Changelog * Type __consul: Add new consul versions (Nico Schottelius) * Type __apt_ppa: Do not install legacy package python-software-properties (Steven Armstrong) - 3.1.13: 2015-05-16 * Type __block: Fix support for non stdin blocks (Dominique Roux) * Type __consul: Install package unzip (Nico Schottelius) From 1b37b9fbb1a36601731dcb653ecf4834bc77fa3f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 25 May 2016 07:25:21 +0200 Subject: [PATCH 5/5] Minor sentence fixes. --- docs/man/man1/cdist.text | 2 +- scripts/cdist | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/man/man1/cdist.text b/docs/man/man1/cdist.text index 4e45ae81..1dc1e87f 100644 --- a/docs/man/man1/cdist.text +++ b/docs/man/man1/cdist.text @@ -111,7 +111,7 @@ EXAMPLES --remote-copy /path/to/my/remote/copy \ -p ikq02.ethz.ch ikq03.ethz.ch ikq04.ethz.ch -# Configure hosts in parallel by reading hosts from file loadbalancers +# Configure hosts read from file loadbalancers % cdist config -f loadbalancers # Display banner diff --git a/scripts/cdist b/scripts/cdist index ccdb5232..6baa28f3 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -85,7 +85,7 @@ def commandline(): parser['config'] = parser['sub'].add_parser('config', parents=[parser['loglevel']]) parser['config'].add_argument('host', nargs='*', - help='one or more hosts to operate on') + help='host(s) to operate on') parser['config'].add_argument('-c', '--conf-dir', help=('Add configuration directory (can be repeated, ' 'last one wins)'), action='append')