From 8c53ce78f5cf8607b544d53d51da77cbfe7f6dc4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 10:46:49 +0100 Subject: [PATCH 1/7] Started the good, the bad and the ugly - code cleanup. --- cdist/config.py | 65 ++-------------------------------------- cdist/util/ipaddr.py | 57 +++++++++++++++++++++++++++++++++++ cdist/util/remoteutil.py | 50 +++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 62 deletions(-) create mode 100644 cdist/util/ipaddr.py create mode 100644 cdist/util/remoteutil.py diff --git a/cdist/config.py b/cdist/config.py index b8d0672c..855aaade 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -34,38 +34,10 @@ import cdist import cdist.exec.local import cdist.exec.remote +import cdist.util.ipaddr as ipaddr from cdist import core - - -def inspect_ssh_mux_opts(): - """Inspect whether or not ssh supports multiplexing options. - - Return string containing multiplexing options if supported. - If ControlPath is supported then placeholder for that path is - specified and can be used for final string formatting. - For example, this function can return string: - "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}". - Then it can be formatted: - mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path'). - """ - import subprocess - - wanted_mux_opts = { - "ControlPath": "{}", - "ControlMaster": "auto", - "ControlPersist": "125", - } - mux_opts = " ".join([" -o {}={}".format( - x, wanted_mux_opts[x]) for x in wanted_mux_opts]) - try: - subprocess.check_output("ssh {}".format(mux_opts), - stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as e: - subproc_output = e.output.decode().lower() - if "bad configuration option" in subproc_output: - return "" - return mux_opts +from cdist.util.remoteutil import inspect_ssh_mux_opts class Config(object): @@ -253,38 +225,7 @@ class Config(object): log.debug("remote_copy for host \"{}\": {}".format( host, remote_copy)) - try: - # getaddrinfo returns a list of 5-tuples: - # (family, type, proto, canonname, sockaddr) - # where sockaddr is: - # (address, port) for AF_INET, - # (address, port, flow_info, scopeid) for AF_INET6 - ip_addr = socket.getaddrinfo( - host, None, type=socket.SOCK_STREAM)[0][4][0] - # gethostbyaddr returns triple - # (hostname, aliaslist, ipaddrlist) - host_name = socket.gethostbyaddr(ip_addr)[0] - log.debug("derived host_name for host \"{}\": {}".format( - host, host_name)) - except (socket.gaierror, socket.herror) as e: - log.warn("Could not derive host_name for {}" - ", $host_name will be empty. Error is: {}".format( - host, e)) - # in case of error provide empty value - host_name = '' - - try: - host_fqdn = socket.getfqdn(host) - log.debug("derived host_fqdn for host \"{}\": {}".format( - host, host_fqdn)) - except socket.herror as e: - log.warn("Could not derive host_fqdn for {}" - ", $host_fqdn will be empty. Error is: {}".format( - host, e)) - # in case of error provide empty value - host_fqdn = '' - - target_host = (host, host_name, host_fqdn) + target_host = ipaddr.resolve_target_addresses(host) log.debug("target_host: {}".format(target_host)) local = cdist.exec.local.Local( diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py new file mode 100644 index 00000000..7c3c037a --- /dev/null +++ b/cdist/util/ipaddr.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import socket +import logging + + +def resolve_target_addresses(host): + log = logging.getLogger(host) + try: + # getaddrinfo returns a list of 5-tuples: + # (family, type, proto, canonname, sockaddr) + # where sockaddr is: + # (address, port) for AF_INET, + # (address, port, flow_info, scopeid) for AF_INET6 + ip_addr = socket.getaddrinfo( + host, None, type=socket.SOCK_STREAM)[0][4][0] + # gethostbyaddr returns triple + # (hostname, aliaslist, ipaddrlist) + host_name = socket.gethostbyaddr(ip_addr)[0] + log.debug("derived host_name for host \"{}\": {}".format( + host, host_name)) + except (socket.gaierror, socket.herror) as e: + log.warn("Could not derive host_name for {}" + ", $host_name will be empty. Error is: {}".format(host, e)) + # in case of error provide empty value + host_name = '' + + try: + host_fqdn = socket.getfqdn(host) + log.debug("derived host_fqdn for host \"{}\": {}".format( + host, host_fqdn)) + except socket.herror as e: + log.warn("Could not derive host_fqdn for {}" + ", $host_fqdn will be empty. Error is: {}".format(host, e)) + # in case of error provide empty value + host_fqdn = '' + + return (host, host_name, host_fqdn) diff --git a/cdist/util/remoteutil.py b/cdist/util/remoteutil.py new file mode 100644 index 00000000..c18d6705 --- /dev/null +++ b/cdist/util/remoteutil.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +def inspect_ssh_mux_opts(): + """Inspect whether or not ssh supports multiplexing options. + + Return string containing multiplexing options if supported. + If ControlPath is supported then placeholder for that path is + specified and can be used for final string formatting. + For example, this function can return string: + "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}". + Then it can be formatted: + mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path'). + """ + import subprocess + + wanted_mux_opts = { + "ControlPath": "{}", + "ControlMaster": "auto", + "ControlPersist": "125", + } + mux_opts = " ".join([" -o {}={}".format( + x, wanted_mux_opts[x]) for x in wanted_mux_opts]) + try: + subprocess.check_output("ssh {}".format(mux_opts), + stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + subproc_output = e.output.decode().lower() + if "bad configuration option" in subproc_output: + return "" + return mux_opts From e6fc74c081423c8095a3d1e8e72d8ce24699f436 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 18:12:38 +0100 Subject: [PATCH 2/7] ugly -> bad --- cdist/config.py | 111 ++++++++++++++++++------------------------- cdist/exec/remote.py | 31 +----------- cdist/hostsource.py | 72 ++++++++++++++++++++++++++++ cdist/util/ipaddr.py | 26 ++++++++++ 4 files changed, 146 insertions(+), 94 deletions(-) create mode 100644 cdist/hostsource.py diff --git a/cdist/config.py b/cdist/config.py index 855aaade..b1a120ca 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -22,15 +22,14 @@ import logging import os -import shutil import sys import time -import pprint import itertools import tempfile import socket import cdist +import cdist.hostsource import cdist.exec.local import cdist.exec.remote @@ -61,55 +60,17 @@ class Config(object): self.local.create_files_dirs() self.remote.create_files_dirs() - @staticmethod - def hostfile_process_line(line): - """Return host from read line or None if no host present.""" - if not line: - return None - # remove comment if present - comment_index = line.find('#') - if comment_index >= 0: - host = line[:comment_index] - else: - host = line - # remove leading and trailing whitespaces - host = host.strip() - # skip empty lines - if host: - return host - else: - return None - @staticmethod def hosts(source): - """Yield hosts from source. - Source can be a sequence or filename (stdin if \'-\'). - In case of filename each line represents one host. - """ - if isinstance(source, str): - import fileinput - try: - for host in fileinput.input(files=(source)): - host = Config.hostfile_process_line(host) - if host: - yield host - except (IOError, OSError, UnicodeError) as e: - raise cdist.Error( - "Error reading hosts from file \'{}\': {}".format( - source, e)) - else: - if source: - for host in source: - yield host + try: + yield from cdist.hostsource.HostSource(source)() + except (IOError, OSError, UnicodeError) as e: + raise cdist.Error( + "Error reading hosts from \'{}\': {}".format( + source, e)) @classmethod - def commandline(cls, args): - """Configure remote system""" - import multiprocessing - - # FIXME: Refactor relict - remove later - log = logging.getLogger("cdist") - + def _check_and_prepare_args(cls, args): if args.manifest == '-' and args.hostfile == '-': raise cdist.Error(("Cannot read both, manifest and host file, " "from stdin")) @@ -134,10 +95,6 @@ class Config(object): import atexit atexit.register(lambda: os.remove(initial_manifest_temp_path)) - process = {} - failed_hosts = [] - time_start = time.time() - # default remote cmd patterns args.remote_exec_pattern = None args.remote_copy_pattern = None @@ -154,10 +111,29 @@ class Config(object): if args_dict['remote_copy'] is None: args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts + @classmethod + def _base_root_path(cls, args): if args.out_path: base_root_path = args.out_path else: base_root_path = tempfile.mkdtemp() + return base_root_path + + @classmethod + def commandline(cls, args): + """Configure remote system""" + import multiprocessing + + # FIXME: Refactor relict - remove later + log = logging.getLogger("cdist") + + cls._check_and_prepare_args(args) + + process = {} + failed_hosts = [] + time_start = time.time() + + base_root_path = cls._base_root_path(args) hostcnt = 0 for host in itertools.chain(cls.hosts(args.host), @@ -199,6 +175,24 @@ class Config(object): raise cdist.Error("Failed to configure the following hosts: " + " ".join(failed_hosts)) + @classmethod + def _resolve_remote_cmds(cls, args, host_base_path): + control_path = os.path.join(host_base_path, "ssh-control-path") + # If we constructed patterns for remote commands then there is + # placeholder for ssh ControlPath, format it and we have unique + # ControlPath for each host. + # + # If not then use args.remote_exec/copy that user specified. + if args.remote_exec_pattern: + remote_exec = args.remote_exec_pattern.format(control_path) + else: + remote_exec = args.remote_exec + if args.remote_copy_pattern: + remote_copy = args.remote_copy_pattern.format(control_path) + else: + remote_copy = args.remote_copy + return (remote_exec, remote_copy, ) + @classmethod def onehost(cls, host, host_base_path, host_dir_name, args, parallel): """Configure ONE system""" @@ -206,20 +200,7 @@ class Config(object): log = logging.getLogger(host) try: - control_path = os.path.join(host_base_path, "ssh-control-path") - # If we constructed patterns for remote commands then there is - # placeholder for ssh ControlPath, format it and we have unique - # ControlPath for each host. - # - # If not then use args.remote_exec/copy that user specified. - if args.remote_exec_pattern: - remote_exec = args.remote_exec_pattern.format(control_path) - else: - remote_exec = args.remote_exec - if args.remote_copy_pattern: - remote_copy = args.remote_copy_pattern.format(control_path) - else: - remote_copy = args.remote_copy + remote_exec, remote_copy = cls._resolve_remote_cmds(host_base_path) log.debug("remote_exec for host \"{}\": {}".format( host, remote_exec)) log.debug("remote_copy for host \"{}\": {}".format( diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index f374262f..74a33f73 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -30,40 +30,13 @@ import multiprocessing import cdist import cdist.exec.util as exec_util - - -# check whether addr is IPv6 -try: - # python 3.3+ - import ipaddress - - def _is_ipv6(addr): - try: - return ipaddress.ip_address(addr).version == 6 - except ValueError: - return False -except ImportError: - # fallback for older python versions - import socket - - def _is_ipv6(addr): - try: - socket.inet_aton(addr) - return False - except socket.error: - pass - try: - socket.inet_pton(socket.AF_INET6, addr) - return True - except socket.error: - pass - return False +import cdist.util.ipaddr as ipaddr def _wrap_addr(addr): """If addr is IPv6 then return addr wrapped between '[' and ']', otherwise return it intact.""" - if _is_ipv6(addr): + if ipaddr.is_ipv6(addr): return "".join(("[", addr, "]", )) else: return addr diff --git a/cdist/hostsource.py b/cdist/hostsource.py new file mode 100644 index 00000000..8170536c --- /dev/null +++ b/cdist/hostsource.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import fileinput + + +class HostSource(object): + """ + Host source object. + Source can be a sequence or filename (stdin if \'-\'). + In case of filename each line represents one host. + """ + def __init__(self, source): + self.source = source + + def _process_file_line(self, line): + """Return host from read line or None if no host present.""" + if not line: + return None + # remove comment if present + comment_index = line.find('#') + if comment_index >= 0: + host = line[:comment_index] + else: + host = line + # remove leading and trailing whitespaces + host = host.strip() + # skip empty lines + if host: + return host + else: + return None + + def _hosts_from_sequence(self): + for host in self.source: + yield host + + def _hosts_from_file(self): + for line in fileinput.input(files=(self.source)): + host = self._process_file_line(line) + if host: + yield host + + def hosts(self): + if not source: + return + + if isinstance(self.source, str): + yield from self._hosts_from_file() + else: + yield from self._hosts_from_sequence() + + def __call__(self): + yield from self.hosts() diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 7c3c037a..71477682 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -55,3 +55,29 @@ def resolve_target_addresses(host): host_fqdn = '' return (host, host_name, host_fqdn) + + +# check whether addr is IPv6 +try: + # python 3.3+ + import ipaddress + + def is_ipv6(addr): + try: + return ipaddress.ip_address(addr).version == 6 + except ValueError: + return False +except ImportError: + # fallback for older python versions + def is_ipv6(addr): + try: + socket.inet_aton(addr) + return False + except socket.error: + pass + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + pass + return False From d0f5d2c45967429d86002e5526b2bb60cadd3ed6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 18:24:37 +0100 Subject: [PATCH 3/7] ugly -> bad --- cdist/argparse.py | 209 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/cdist | 176 ++------------------------------------ 2 files changed, 218 insertions(+), 167 deletions(-) create mode 100644 cdist/argparse.py mode change 100755 => 100644 scripts/cdist diff --git a/cdist/argparse.py b/cdist/argparse.py new file mode 100644 index 00000000..cef3dd5f --- /dev/null +++ b/cdist/argparse.py @@ -0,0 +1,209 @@ +import argparse +import cdist +import multiprocessing +import os +import logging +import collections + + +# list of beta sub-commands +BETA_COMMANDS = ['install', ] +# list of beta arguments for sub-commands +BETA_ARGS = { + 'config': ['jobs', ], +} +EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" +# Parser others can reuse +parser = None + + +_verbosity_level = { + 0: logging.ERROR, + 1: logging.WARNING, + 2: logging.INFO, +} +_verbosity_level = collections.defaultdict( + lambda: logging.DEBUG, _verbosity_level) + + +def add_beta_command(cmd): + if cmd not in BETA_COMMANDS: + BETA_COMMANDS.append(cmd) + + +def add_beta_arg(cmd, arg): + if cmd in BETA_ARGS: + if arg not in BETA_ARGS[cmd]: + BETA_ARGS[cmd].append(arg) + else: + BETA_ARGS[cmd] = [arg, ] + + +def check_beta(args_dict): + if 'beta' not in args_dict: + args_dict['beta'] = False + # Check only if beta is not enabled: if beta option is specified then + # raise error. + if not args_dict['beta']: + cmd = args_dict['command'] + # first check if command is beta + if cmd in BETA_COMMANDS: + raise cdist.CdistBetaRequired(cmd) + # then check if some command's argument is beta + if cmd in BETA_ARGS: + for arg in BETA_ARGS[cmd]: + if arg in args_dict and args_dict[arg]: + raise cdist.CdistBetaRequired(cmd, arg) + + +def check_positive_int(value): + import argparse + + try: + val = int(value) + except ValueError: + raise argparse.ArgumentTypeError( + "{} is invalid int value".format(value)) + if val <= 0: + raise argparse.ArgumentTypeError( + "{} is invalid positive int value".format(val)) + return val + + +def get_parsers(): + global parser + + # Construct parser others can reuse + if parser: + return parser + else: + parser = {} + # Options _all_ parsers have in common + parser['loglevel'] = argparse.ArgumentParser(add_help=False) + parser['loglevel'].add_argument( + '-d', '--debug', + help=('Set log level to debug (deprecated, use -vvv instead)'), + action='store_true', default=False) + parser['loglevel'].add_argument( + '-v', '--verbose', + help=('Increase log level, be more verbose. Use it more than once ' + 'to increase log level. The order of levels from the lowest ' + 'to the highest are: ERROR, WARNING, INFO, DEBUG.'), + action='count', default=0) + + parser['beta'] = argparse.ArgumentParser(add_help=False) + parser['beta'].add_argument( + '-b', '--beta', + help=('Enable beta functionalities. ' + 'Can also be enabled using CDIST_BETA env var.'), + action='store_true', dest='beta', + default='CDIST_BETA' in os.environ) + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser( + description='cdist ' + cdist.VERSION, parents=[parser['loglevel']]) + parser['main'].add_argument( + '-V', '--version', help='Show version', action='version', + version='%(prog)s ' + cdist.VERSION) + parser['sub'] = parser['main'].add_subparsers( + title="Commands", dest="command") + + # Banner + parser['banner'] = parser['sub'].add_parser( + 'banner', parents=[parser['loglevel']]) + parser['banner'].set_defaults(func=cdist.banner.banner) + + # Config + parser['config_main'] = argparse.ArgumentParser(add_help=False) + parser['config_main'].add_argument( + '-c', '--conf-dir', + help=('Add configuration directory (can be repeated, ' + 'last one wins)'), action='append') + parser['config_main'].add_argument( + '-i', '--initial-manifest', + help='path to a cdist manifest or \'-\' to read from stdin.', + dest='manifest', required=False) + parser['config_main'].add_argument( + '-j', '--jobs', nargs='?', + type=check_positive_int, + help=('Specify the maximum number of parallel jobs, currently ' + 'only global explorers are supported'), + action='store', dest='jobs', + const=multiprocessing.cpu_count()) + parser['config_main'].add_argument( + '-n', '--dry-run', + help='do not execute code', action='store_true') + parser['config_main'].add_argument( + '-o', '--out-dir', + help='directory to save cdist output in', dest="out_path") + + # remote-copy and remote-exec defaults are environment variables + # if set; if not then None - these will be futher handled after + # parsing to determine implementation default + parser['config_main'].add_argument( + '--remote-copy', + help='Command to use for remote copy (should behave like scp)', + action='store', dest='remote_copy', + default=os.environ.get('CDIST_REMOTE_COPY')) + parser['config_main'].add_argument( + '--remote-exec', + help=('Command to use for remote execution ' + '(should behave like ssh)'), + action='store', dest='remote_exec', + default=os.environ.get('CDIST_REMOTE_EXEC')) + + # Config + parser['config_args'] = argparse.ArgumentParser(add_help=False) + parser['config_args'].add_argument( + 'host', nargs='*', help='host(s) to operate on') + parser['config_args'].add_argument( + '-f', '--file', + help=('Read additional hosts to operate on from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read hosts from stdin.'), + dest='hostfile', required=False) + parser['config_args'].add_argument( + '-p', '--parallel', + help='operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['config_args'].add_argument( + '-s', '--sequential', + help='operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + parser['config'] = parser['sub'].add_parser( + 'config', parents=[parser['loglevel'], parser['beta'], + parser['config_main'], + parser['config_args']]) + parser['config'].set_defaults(func=cdist.config.Config.commandline) + + # Install + parser['install'] = parser['sub'].add_parser('install', add_help=False, + parents=[parser['config']]) + parser['install'].set_defaults(func=cdist.install.Install.commandline) + + # Shell + parser['shell'] = parser['sub'].add_parser( + 'shell', parents=[parser['loglevel']]) + parser['shell'].add_argument( + '-s', '--shell', + help=('Select shell to use, defaults to current shell. Used shell' + ' should be POSIX compatible shell.')) + parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) + + for p in parser: + parser[p].epilog = EPILOG + + return parser + + +def handle_loglevel(args): + if args.debug: + retval = "-d/--debug is deprecated, use -vvv instead" + args.verbose = 3 + else: + retval = None + + logging.root.setLevel(_verbosity_level[args.verbose]) + + return retval diff --git a/scripts/cdist b/scripts/cdist old mode 100755 new mode 100644 index 4ff64bb5..498091b8 --- a/scripts/cdist +++ b/scripts/cdist @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. @@ -24,182 +24,25 @@ import collections import logging -# list of beta sub-commands -BETA_COMMANDS = ['install', ] -# list of beta arguments for sub-commands -BETA_ARGS = { - 'config': ['jobs', ], -} - - -def check_positive_int(value): - import argparse - - try: - val = int(value) - except ValueError as e: - raise argparse.ArgumentTypeError( - "{} is invalid int value".format(value)) - if val <= 0: - raise argparse.ArgumentTypeError( - "{} is invalid positive int value".format(val)) - return val - - -def check_beta(args_dict): - if 'beta' not in args_dict: - args_dict['beta'] = False - # Check only if beta is not enabled: if beta option is specified then - # raise error. - if not args_dict['beta']: - cmd = args_dict['command'] - # first check if command is beta - if cmd in BETA_COMMANDS: - raise cdist.CdistBetaRequired(cmd) - # then check if command's argument is beta - if cmd in BETA_ARGS: - for arg in BETA_ARGS[cmd]: - if arg in args_dict and args_dict[arg]: - raise cdist.CdistBetaRequired(cmd, arg) - - -_verbosity_level = { - 0: logging.ERROR, - 1: logging.WARNING, - 2: logging.INFO, -} -_verbosity_level = collections.defaultdict( - lambda: logging.DEBUG, _verbosity_level) - def commandline(): """Parse command line""" - import argparse + import cdist.argparse import cdist.banner import cdist.config import cdist.install import cdist.shell import shutil import os - import multiprocessing - - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['loglevel'] = argparse.ArgumentParser(add_help=False) - parser['loglevel'].add_argument( - '-d', '--debug', - help=('Set log level to debug (deprecated, use -vvv instead)'), - action='store_true', default=False) - parser['loglevel'].add_argument( - '-v', '--verbose', - help=('Increase log level, be more verbose. Use it more than once ' - 'to increase log level. The order of levels from the lowest ' - 'to the highest are: ERROR, WARNING, INFO, DEBUG.'), - action='count', default=0) - - # Main subcommand parser - parser['main'] = argparse.ArgumentParser( - description='cdist ' + cdist.VERSION, parents=[parser['loglevel']]) - parser['main'].add_argument( - '-V', '--version', help='Show version', action='version', - version='%(prog)s ' + cdist.VERSION) - parser['sub'] = parser['main'].add_subparsers( - title="Commands", dest="command") - - # Banner - parser['banner'] = parser['sub'].add_parser( - 'banner', parents=[parser['loglevel']]) - parser['banner'].set_defaults(func=cdist.banner.banner) - - # Config - parser['config'] = parser['sub'].add_parser( - 'config', parents=[parser['loglevel']]) - parser['config'].add_argument( - 'host', nargs='*', help='host(s) to operate on') - parser['config'].add_argument( - '-b', '--enable-beta', - help=('Enable beta functionalities. Beta functionalities ' - 'include the following options: -j/--jobs.'), - action='store_true', dest='beta', default=False) - parser['config'].add_argument( - '-c', '--conf-dir', - help=('Add configuration directory (can be repeated, ' - 'last one wins)'), action='append') - parser['config'].add_argument( - '-f', '--file', - help=('Read additional hosts to operate on from specified file ' - 'or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read hosts from stdin.'), - dest='hostfile', required=False) - parser['config'].add_argument( - '-i', '--initial-manifest', - help='Path to a cdist manifest or \'-\' to read from stdin.', - dest='manifest', required=False) - parser['config'].add_argument( - '-j', '--jobs', nargs='?', type=check_positive_int, - help=('Specify the maximum number of parallel jobs, currently ' - 'only global explorers are supported (currently in beta'), - action='store', dest='jobs', - const=multiprocessing.cpu_count()) - parser['config'].add_argument( - '-n', '--dry-run', - help='Do not execute code', action='store_true') - parser['config'].add_argument( - '-o', '--out-dir', - help='Directory to save cdist output in', dest="out_path") - parser['config'].add_argument( - '-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['config'].add_argument( - '-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - # remote-copy and remote-exec defaults are environment variables - # if set; if not then None - these will be futher handled after - # parsing to determine implementation default - parser['config'].add_argument( - '--remote-copy', - help='Command to use for remote copy (should behave like scp)', - action='store', dest='remote_copy', - default=os.environ.get('CDIST_REMOTE_COPY')) - parser['config'].add_argument( - '--remote-exec', - help=('Command to use for remote execution ' - '(should behave like ssh)'), - action='store', dest='remote_exec', - default=os.environ.get('CDIST_REMOTE_EXEC')) - parser['config'].set_defaults(func=cdist.config.Config.commandline) - - # Install - parser['install'] = parser['sub'].add_parser('install', add_help=False, - parents=[parser['config']]) - parser['install'].set_defaults(func=cdist.install.Install.commandline) - - # Shell - parser['shell'] = parser['sub'].add_parser( - 'shell', parents=[parser['loglevel']]) - parser['shell'].add_argument( - '-s', '--shell', - help=('Select shell to use, defaults to current shell. Used shell' - ' should be POSIX compatible shell.')) - parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) - - for p in parser: - parser[p].epilog = ( - "Get cdist at http://www.nico.schottelius.org/software/cdist/") + parser = cdist.argparse.get_parsers() args = parser['main'].parse_args(sys.argv[1:]) # Loglevels are handled globally in here - if args.debug: - log.warning("-d/--debug is deprecated, use -vvv instead") - args.verbose = 3 - - logging.root.setLevel(_verbosity_level[args.verbose]) + retval = cdist.argparse.handle_loglevel(args) + if retval: + log.warning(retval) log.debug(args) log.info("version %s" % cdist.VERSION) @@ -219,17 +62,16 @@ def commandline(): parser['main'].print_help() sys.exit(0) - check_beta(vars(args)) + cdist.argparse.check_beta(vars(args)) args.func(args) if __name__ == "__main__": - # Sys is needed for sys.exit() import sys cdistpythonversion = '3.2' if sys.version < cdistpythonversion: - print('Python >= ' + cdistpythonversion + - ' is required on the source host.', file=sys.stderr) + print('Python >= {} is required on the source host.'.format( + cdistpythonversion), file=sys.stderr) sys.exit(1) exit_code = 0 From 2999f1269873245733a612aac82c6e768eba7663 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 18:29:10 +0100 Subject: [PATCH 4/7] chmod +x --- scripts/cdist | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/cdist diff --git a/scripts/cdist b/scripts/cdist old mode 100644 new mode 100755 From 609977b7ff8d5659622f3fff8494b8fd8b9aa427 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 4 Dec 2016 20:27:42 +0100 Subject: [PATCH 5/7] ugly->bad --- cdist/__init__.py | 4 +- cdist/core/explorer.py | 11 ++--- cdist/exec/remote.py | 80 +++++++++++++++++---------------- cdist/test/exec/remote.py | 4 +- docs/src/cdist-reference.rst.sh | 3 ++ docs/src/man1/cdist.rst | 3 ++ 6 files changed, 54 insertions(+), 51 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index b6f5c8cb..c142230c 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error): err_msg = ("\'{}\' command is beta, but beta is " "not enabled. If you want to use it please enable beta " "functionalities by using the -b/--enable-beta command " - "line flag.") + "line flag or setting CDIST_BETA env var.") fmt_args = [self.command, ] else: err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta " "is not enabled. If you want to use it please enable " "beta functionalities by using the -b/--enable-beta " - "command line flag.") + "command line flag or setting CDIST_BETA env var.") fmt_args = [self.arg, self.command, ] return err_msg.format(*fmt_args) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index ef85431c..23996240 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -149,14 +149,9 @@ class Explorer(object): def transfer_global_explorers(self): """Transfer the global explorers to the remote side.""" self.remote.mkdir(self.remote.global_explorer_path) - if self.jobs is None: - self.remote.transfer(self.local.global_explorer_path, - self.remote.global_explorer_path) - else: - self.remote.transfer_dir_parallel( - self.local.global_explorer_path, - self.remote.global_explorer_path, - self.jobs) + self.remote.transfer(self.local.global_explorer_path, + self.remote.global_explorer_path, + self.jobs) self.remote.run(["chmod", "0700", "%s/*" % (self.remote.global_explorer_path)]) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 74a33f73..440aafa7 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -118,57 +118,59 @@ class Remote(object): self.log.debug("Remote mkdir: %s", path) self.run(["mkdir", "-p", path]) - def transfer(self, source, destination): + def transfer(self, source, destination, jobs=None): """Transfer a file or directory to the remote side.""" self.log.debug("Remote transfer: %s -> %s", source, destination) self.rmdir(destination) if os.path.isdir(source): self.mkdir(destination) - for f in glob.glob1(source, '*'): - command = self._copy.split() - path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - self._run_command(command) + if jobs: + self._transfer_dir_parallel(source, destination, jobs) + else: + self._transfer_dir_sequential(source, destination) + elif jobs: + raise cdist.Error("Source {} is not a directory".format(source)) else: command = self._copy.split() command.extend([source, '{0}:{1}'.format( _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) - def transfer_dir_parallel(self, source, destination, jobs): - """Transfer a directory to the remote side in parallel mode.""" - self.log.debug("Remote transfer: %s -> %s", source, destination) - self.rmdir(destination) - if os.path.isdir(source): - self.mkdir(destination) - self.log.info("Remote transfer in {} parallel jobs".format( - jobs)) - self.log.debug("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for parallel " - "remote transfer")) - with multiprocessing.Pool(jobs) as pool: - self.log.debug("Starting async for parallel transfer") - commands = [] - for f in glob.glob1(source, '*'): - command = self._copy.split() - path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - commands.append(command) - results = [ - pool.apply_async(self._run_command, (cmd,)) - for cmd in commands - ] + def _transfer_dir_sequential(self, source, destination): + for f in glob.glob1(source, '*'): + command = self._copy.split() + path = os.path.join(source, f) + command.extend([path, '{0}:{1}'.format( + _wrap_addr(self.target_host[0]), destination)]) + self._run_command(command) - self.log.debug("Waiting async results for parallel transfer") - for r in results: - r.get() # self._run_command returns None - self.log.debug(("Multiprocessing for parallel transfer " - "finished")) - else: - raise cdist.Error("Source {} is not a directory".format(source)) + def _transfer_dir_parallel(self, source, destination, jobs): + """Transfer a directory to the remote side in parallel mode.""" + self.log.info("Remote transfer in {} parallel jobs".format( + jobs)) + self.log.debug("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.debug(("Starting multiprocessing Pool for parallel " + "remote transfer")) + with multiprocessing.Pool(jobs) as pool: + self.log.debug("Starting async for parallel transfer") + commands = [] + for f in glob.glob1(source, '*'): + command = self._copy.split() + path = os.path.join(source, f) + command.extend([path, '{0}:{1}'.format( + _wrap_addr(self.target_host[0]), destination)]) + commands.append(command) + results = [ + pool.apply_async(self._run_command, (cmd,)) + for cmd in commands + ] + + self.log.debug("Waiting async results for parallel transfer") + for r in results: + r.get() # self._run_command returns None + self.log.debug(("Multiprocessing for parallel transfer " + "finished")) def run_script(self, script, env=None, return_output=False): """Run the given script with the given environment on the remote side. diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index 45dabb18..371d17e3 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase): source_file_name = os.path.split(source_file)[-1] filenames.append(source_file_name) target = self.mkdtemp(dir=self.temp_dir) - self.remote.transfer_dir_parallel(source, target, - multiprocessing.cpu_count()) + self.remote.transfer(source, target, + multiprocessing.cpu_count()) # test if the payload files are in the target directory for filename in filenames: self.assertTrue(os.path.isfile(os.path.join(target, filename))) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 97b22473..4b94b858 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). + +CDIST_BETA + Enable beta functionalities. eof diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 5daedcd4..08c856b1 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). +CDIST_BETA + Enable beta functionalities. + EXIT STATUS ----------- The following exit values shall be returned: From 3e763e9e6c9c07398df10510b11e1167f755d764 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Dec 2016 18:36:19 +0100 Subject: [PATCH 6/7] list -> set for beta commands and args --- cdist/argparse.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index cef3dd5f..04f6e6a4 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -6,11 +6,11 @@ import logging import collections -# list of beta sub-commands -BETA_COMMANDS = ['install', ] -# list of beta arguments for sub-commands +# set of beta sub-commands +BETA_COMMANDS = set(('install', )) +# set of beta arguments for sub-commands BETA_ARGS = { - 'config': ['jobs', ], + 'config': set(('jobs', )), } EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" # Parser others can reuse @@ -27,8 +27,7 @@ _verbosity_level = collections.defaultdict( def add_beta_command(cmd): - if cmd not in BETA_COMMANDS: - BETA_COMMANDS.append(cmd) + BETA_COMMANDS.add(cmd) def add_beta_arg(cmd, arg): @@ -36,7 +35,7 @@ def add_beta_arg(cmd, arg): if arg not in BETA_ARGS[cmd]: BETA_ARGS[cmd].append(arg) else: - BETA_ARGS[cmd] = [arg, ] + BETA_ARGS[cmd] = set((arg, )) def check_beta(args_dict): From 341de216a63e8ae2922fa3aa375026a65d8f52e3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Dec 2016 18:39:43 +0100 Subject: [PATCH 7/7] Fix missing vars. --- cdist/config.py | 3 ++- cdist/hostsource.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b1a120ca..6b57e7bf 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -200,7 +200,8 @@ class Config(object): log = logging.getLogger(host) try: - remote_exec, remote_copy = cls._resolve_remote_cmds(host_base_path) + remote_exec, remote_copy = cls._resolve_remote_cmds( + args, host_base_path) log.debug("remote_exec for host \"{}\": {}".format( host, remote_exec)) log.debug("remote_copy for host \"{}\": {}".format( diff --git a/cdist/hostsource.py b/cdist/hostsource.py index 8170536c..9c2c0616 100644 --- a/cdist/hostsource.py +++ b/cdist/hostsource.py @@ -60,7 +60,7 @@ class HostSource(object): yield host def hosts(self): - if not source: + if not self.source: return if isinstance(self.source, str):