Merge pull request #504 from darko-poljak/the-good-the-bad-and-the-ugly

The Good, the Bad and the Ugly

cdist/__init__.py

@@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error):
             err_msg = ("\'{}\' command is beta, but beta is "
                        "not enabled. If you want to use it please enable beta "
                        "functionalities by using the -b/--enable-beta command "
-                       "line flag.")
+                       "line flag or setting CDIST_BETA env var.")
             fmt_args = [self.command, ]
         else:
             err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta "
                        "is not enabled. If you want to use it please enable "
                        "beta functionalities by using the -b/--enable-beta "
-                       "command line flag.")
+                       "command line flag or setting CDIST_BETA env var.")
             fmt_args = [self.arg, self.command, ]
         return err_msg.format(*fmt_args)
 

cdist/argparse.py

@@ -0,0 +1,208 @@
+import argparse
+import cdist
+import multiprocessing
+import os
+import logging
+import collections
+
+
+# set of beta sub-commands
+BETA_COMMANDS = set(('install', ))
+# set of beta arguments for sub-commands
+BETA_ARGS = {
+    'config': set(('jobs', )),
+}
+EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
+# Parser others can reuse
+parser = None
+
+
+_verbosity_level = {
+    0: logging.ERROR,
+    1: logging.WARNING,
+    2: logging.INFO,
+}
+_verbosity_level = collections.defaultdict(
+    lambda: logging.DEBUG, _verbosity_level)
+
+
+def add_beta_command(cmd):
+    BETA_COMMANDS.add(cmd)
+
+
+def add_beta_arg(cmd, arg):
+    if cmd in BETA_ARGS:
+        if arg not in BETA_ARGS[cmd]:
+            BETA_ARGS[cmd].append(arg)
+    else:
+        BETA_ARGS[cmd] = set((arg, ))
+
+
+def check_beta(args_dict):
+    if 'beta' not in args_dict:
+        args_dict['beta'] = False
+    # Check only if beta is not enabled: if beta option is specified then
+    # raise error.
+    if not args_dict['beta']:
+        cmd = args_dict['command']
+        # first check if command is beta
+        if cmd in BETA_COMMANDS:
+            raise cdist.CdistBetaRequired(cmd)
+        # then check if some command's argument is beta
+        if cmd in BETA_ARGS:
+            for arg in BETA_ARGS[cmd]:
+                if arg in args_dict and args_dict[arg]:
+                    raise cdist.CdistBetaRequired(cmd, arg)
+
+
+def check_positive_int(value):
+    import argparse
+
+    try:
+        val = int(value)
+    except ValueError:
+        raise argparse.ArgumentTypeError(
+                "{} is invalid int value".format(value))
+    if val <= 0:
+        raise argparse.ArgumentTypeError(
+                "{} is invalid positive int value".format(val))
+    return val
+
+
+def get_parsers():
+    global parser
+
+    # Construct parser others can reuse
+    if parser:
+        return parser
+    else:
+        parser = {}
+    # Options _all_ parsers have in common
+    parser['loglevel'] = argparse.ArgumentParser(add_help=False)
+    parser['loglevel'].add_argument(
+            '-d', '--debug',
+            help=('Set log level to debug (deprecated, use -vvv instead)'),
+            action='store_true', default=False)
+    parser['loglevel'].add_argument(
+            '-v', '--verbose',
+            help=('Increase log level, be more verbose. Use it more than once '
+                  'to increase log level. The order of levels from the lowest '
+                  'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
+            action='count', default=0)
+
+    parser['beta'] = argparse.ArgumentParser(add_help=False)
+    parser['beta'].add_argument(
+           '-b', '--beta',
+           help=('Enable beta functionalities. '
+                 'Can also be enabled using CDIST_BETA env var.'),
+           action='store_true', dest='beta',
+           default='CDIST_BETA' in os.environ)
+
+    # Main subcommand parser
+    parser['main'] = argparse.ArgumentParser(
+            description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
+    parser['main'].add_argument(
+            '-V', '--version', help='Show version', action='version',
+            version='%(prog)s ' + cdist.VERSION)
+    parser['sub'] = parser['main'].add_subparsers(
+            title="Commands", dest="command")
+
+    # Banner
+    parser['banner'] = parser['sub'].add_parser(
+            'banner', parents=[parser['loglevel']])
+    parser['banner'].set_defaults(func=cdist.banner.banner)
+
+    # Config
+    parser['config_main'] = argparse.ArgumentParser(add_help=False)
+    parser['config_main'].add_argument(
+            '-c', '--conf-dir',
+            help=('Add configuration directory (can be repeated, '
+                  'last one wins)'), action='append')
+    parser['config_main'].add_argument(
+           '-i', '--initial-manifest',
+           help='path to a cdist manifest or \'-\' to read from stdin.',
+           dest='manifest', required=False)
+    parser['config_main'].add_argument(
+           '-j', '--jobs', nargs='?',
+           type=check_positive_int,
+           help=('Specify the maximum number of parallel jobs, currently '
+                 'only global explorers are supported'),
+           action='store', dest='jobs',
+           const=multiprocessing.cpu_count())
+    parser['config_main'].add_argument(
+           '-n', '--dry-run',
+           help='do not execute code', action='store_true')
+    parser['config_main'].add_argument(
+           '-o', '--out-dir',
+           help='directory to save cdist output in', dest="out_path")
+
+    # remote-copy and remote-exec defaults are environment variables
+    # if set; if not then None - these will be futher handled after
+    # parsing to determine implementation default
+    parser['config_main'].add_argument(
+           '--remote-copy',
+           help='Command to use for remote copy (should behave like scp)',
+           action='store', dest='remote_copy',
+           default=os.environ.get('CDIST_REMOTE_COPY'))
+    parser['config_main'].add_argument(
+           '--remote-exec',
+           help=('Command to use for remote execution '
+                 '(should behave like ssh)'),
+           action='store', dest='remote_exec',
+           default=os.environ.get('CDIST_REMOTE_EXEC'))
+
+    # Config
+    parser['config_args'] = argparse.ArgumentParser(add_help=False)
+    parser['config_args'].add_argument(
+            'host', nargs='*', help='host(s) to operate on')
+    parser['config_args'].add_argument(
+            '-f', '--file',
+            help=('Read additional hosts to operate on from specified file '
+                  'or from stdin if \'-\' (each host on separate line). '
+                  'If no host or host file is specified then, by default, '
+                  'read hosts from stdin.'),
+            dest='hostfile', required=False)
+    parser['config_args'].add_argument(
+           '-p', '--parallel',
+           help='operate on multiple hosts in parallel',
+           action='store_true', dest='parallel')
+    parser['config_args'].add_argument(
+           '-s', '--sequential',
+           help='operate on multiple hosts sequentially (default)',
+           action='store_false', dest='parallel')
+    parser['config'] = parser['sub'].add_parser(
+            'config', parents=[parser['loglevel'], parser['beta'],
+                               parser['config_main'],
+                               parser['config_args']])
+    parser['config'].set_defaults(func=cdist.config.Config.commandline)
+
+    # Install
+    parser['install'] = parser['sub'].add_parser('install', add_help=False,
+                                                 parents=[parser['config']])
+    parser['install'].set_defaults(func=cdist.install.Install.commandline)
+
+    # Shell
+    parser['shell'] = parser['sub'].add_parser(
+            'shell', parents=[parser['loglevel']])
+    parser['shell'].add_argument(
+            '-s', '--shell',
+            help=('Select shell to use, defaults to current shell. Used shell'
+                  ' should be POSIX compatible shell.'))
+    parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
+
+    for p in parser:
+        parser[p].epilog = EPILOG
+
+    return parser
+
+
+def handle_loglevel(args):
+    if args.debug:
+        retval = "-d/--debug is deprecated, use -vvv instead"
+        args.verbose = 3
+    else:
+        retval = None
+
+    logging.root.setLevel(_verbosity_level[args.verbose])
+
+    return retval

cdist/config.py

@@ -22,50 +22,21 @@
 
 import logging
 import os
-import shutil
 import sys
 import time
-import pprint
 import itertools
 import tempfile
 import socket
 
 import cdist
+import cdist.hostsource
 
 import cdist.exec.local
 import cdist.exec.remote
+import cdist.util.ipaddr as ipaddr
 
 from cdist import core
-
+from cdist.util.remoteutil import inspect_ssh_mux_opts
-
-def inspect_ssh_mux_opts():
-    """Inspect whether or not ssh supports multiplexing options.
-
-       Return string containing multiplexing options if supported.
-       If ControlPath is supported then placeholder for that path is
-       specified and can be used for final string formatting.
-       For example, this function can return string:
-       "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
-       Then it can be formatted:
-       mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
-    """
-    import subprocess
-
-    wanted_mux_opts = {
-        "ControlPath": "{}",
-        "ControlMaster": "auto",
-        "ControlPersist": "125",
-    }
-    mux_opts = " ".join([" -o {}={}".format(
-        x, wanted_mux_opts[x]) for x in wanted_mux_opts])
-    try:
-        subprocess.check_output("ssh {}".format(mux_opts),
-                                stderr=subprocess.STDOUT, shell=True)
-    except subprocess.CalledProcessError as e:
-        subproc_output = e.output.decode().lower()
-        if "bad configuration option" in subproc_output:
-            return ""
-    return mux_opts
 
 
 class Config(object):
@@ -89,55 +60,17 @@ class Config(object):
         self.local.create_files_dirs()
         self.remote.create_files_dirs()
 
-    @staticmethod
-    def hostfile_process_line(line):
-        """Return host from read line or None if no host present."""
-        if not line:
-            return None
-        # remove comment if present
-        comment_index = line.find('#')
-        if comment_index >= 0:
-            host = line[:comment_index]
-        else:
-            host = line
-        # remove leading and trailing whitespaces
-        host = host.strip()
-        # skip empty lines
-        if host:
-            return host
-        else:
-            return None
-
     @staticmethod
     def hosts(source):
-        """Yield hosts from source.
+        try:
-           Source can be a sequence or filename (stdin if \'-\').
+            yield from cdist.hostsource.HostSource(source)()
-           In case of filename each line represents one host.
+        except (IOError, OSError, UnicodeError) as e:
-        """
+            raise cdist.Error(
-        if isinstance(source, str):
+                    "Error reading hosts from \'{}\': {}".format(
-            import fileinput
+                        source, e))
-            try:
-                for host in fileinput.input(files=(source)):
-                    host = Config.hostfile_process_line(host)
-                    if host:
-                        yield host
-            except (IOError, OSError, UnicodeError) as e:
-                raise cdist.Error(
-                        "Error reading hosts from file \'{}\': {}".format(
-                            source, e))
-        else:
-            if source:
-                for host in source:
-                    yield host
 
     @classmethod
-    def commandline(cls, args):
+    def _check_and_prepare_args(cls, args):
-        """Configure remote system"""
-        import multiprocessing
-
-        # FIXME: Refactor relict - remove later
-        log = logging.getLogger("cdist")
-
         if args.manifest == '-' and args.hostfile == '-':
             raise cdist.Error(("Cannot read both, manifest and host file, "
                                "from stdin"))
@@ -162,10 +95,6 @@ class Config(object):
             import atexit
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
 
-        process = {}
-        failed_hosts = []
-        time_start = time.time()
-
         # default remote cmd patterns
         args.remote_exec_pattern = None
         args.remote_copy_pattern = None
@@ -182,10 +111,29 @@ class Config(object):
             if args_dict['remote_copy'] is None:
                 args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
 
+    @classmethod
+    def _base_root_path(cls, args):
         if args.out_path:
             base_root_path = args.out_path
         else:
             base_root_path = tempfile.mkdtemp()
+        return base_root_path
+
+    @classmethod
+    def commandline(cls, args):
+        """Configure remote system"""
+        import multiprocessing
+
+        # FIXME: Refactor relict - remove later
+        log = logging.getLogger("cdist")
+
+        cls._check_and_prepare_args(args)
+
+        process = {}
+        failed_hosts = []
+        time_start = time.time()
+
+        base_root_path = cls._base_root_path(args)
 
         hostcnt = 0
         for host in itertools.chain(cls.hosts(args.host),
@@ -227,6 +175,24 @@ class Config(object):
             raise cdist.Error("Failed to configure the following hosts: " +
                               " ".join(failed_hosts))
 
+    @classmethod
+    def _resolve_remote_cmds(cls, args, host_base_path):
+        control_path = os.path.join(host_base_path, "ssh-control-path")
+        # If we constructed patterns for remote commands then there is
+        # placeholder for ssh ControlPath, format it and we have unique
+        # ControlPath for each host.
+        #
+        # If not then use args.remote_exec/copy that user specified.
+        if args.remote_exec_pattern:
+            remote_exec = args.remote_exec_pattern.format(control_path)
+        else:
+            remote_exec = args.remote_exec
+        if args.remote_copy_pattern:
+            remote_copy = args.remote_copy_pattern.format(control_path)
+        else:
+            remote_copy = args.remote_copy
+        return (remote_exec, remote_copy, )
+
     @classmethod
     def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
         """Configure ONE system"""
@@ -234,57 +200,14 @@ class Config(object):
         log = logging.getLogger(host)
 
         try:
-            control_path = os.path.join(host_base_path, "ssh-control-path")
+            remote_exec, remote_copy = cls._resolve_remote_cmds(
-            # If we constructed patterns for remote commands then there is
+                args, host_base_path)
-            # placeholder for ssh ControlPath, format it and we have unique
-            # ControlPath for each host.
-            #
-            # If not then use args.remote_exec/copy that user specified.
-            if args.remote_exec_pattern:
-                remote_exec = args.remote_exec_pattern.format(control_path)
-            else:
-                remote_exec = args.remote_exec
-            if args.remote_copy_pattern:
-                remote_copy = args.remote_copy_pattern.format(control_path)
-            else:
-                remote_copy = args.remote_copy
             log.debug("remote_exec for host \"{}\": {}".format(
                 host, remote_exec))
             log.debug("remote_copy for host \"{}\": {}".format(
                 host, remote_copy))
 
-            try:
+            target_host = ipaddr.resolve_target_addresses(host)
-                # getaddrinfo returns a list of 5-tuples:
-                # (family, type, proto, canonname, sockaddr)
-                # where sockaddr is:
-                # (address, port) for AF_INET,
-                # (address, port, flow_info, scopeid) for AF_INET6
-                ip_addr = socket.getaddrinfo(
-                        host, None, type=socket.SOCK_STREAM)[0][4][0]
-                # gethostbyaddr returns triple
-                # (hostname, aliaslist, ipaddrlist)
-                host_name = socket.gethostbyaddr(ip_addr)[0]
-                log.debug("derived host_name for host \"{}\": {}".format(
-                    host, host_name))
-            except (socket.gaierror, socket.herror) as e:
-                log.warn("Could not derive host_name for {}"
-                         ", $host_name will be empty. Error is: {}".format(
-                             host, e))
-                # in case of error provide empty value
-                host_name = ''
-
-            try:
-                host_fqdn = socket.getfqdn(host)
-                log.debug("derived host_fqdn for host \"{}\": {}".format(
-                    host, host_fqdn))
-            except socket.herror as e:
-                log.warn("Could not derive host_fqdn for {}"
-                         ", $host_fqdn will be empty. Error is: {}".format(
-                             host, e))
-                # in case of error provide empty value
-                host_fqdn = ''
-
-            target_host = (host, host_name, host_fqdn)
             log.debug("target_host: {}".format(target_host))
 
             local = cdist.exec.local.Local(

cdist/core/explorer.py

@@ -149,14 +149,9 @@ class Explorer(object):
     def transfer_global_explorers(self):
         """Transfer the global explorers to the remote side."""
         self.remote.mkdir(self.remote.global_explorer_path)
-        if self.jobs is None:
+        self.remote.transfer(self.local.global_explorer_path,
-            self.remote.transfer(self.local.global_explorer_path,
+                             self.remote.global_explorer_path,
-                                 self.remote.global_explorer_path)
+                             self.jobs)
-        else:
-            self.remote.transfer_dir_parallel(
-                    self.local.global_explorer_path,
-                    self.remote.global_explorer_path,
-                    self.jobs)
         self.remote.run(["chmod", "0700",
                          "%s/*" % (self.remote.global_explorer_path)])
 

cdist/exec/remote.py

@@ -30,40 +30,13 @@ import multiprocessing
 
 import cdist
 import cdist.exec.util as exec_util
-
+import cdist.util.ipaddr as ipaddr
-
-# check whether addr is IPv6
-try:
-    # python 3.3+
-    import ipaddress
-
-    def _is_ipv6(addr):
-        try:
-            return ipaddress.ip_address(addr).version == 6
-        except ValueError:
-            return False
-except ImportError:
-    # fallback for older python versions
-    import socket
-
-    def _is_ipv6(addr):
-        try:
-            socket.inet_aton(addr)
-            return False
-        except socket.error:
-            pass
-        try:
-            socket.inet_pton(socket.AF_INET6, addr)
-            return True
-        except socket.error:
-            pass
-        return False
 
 
 def _wrap_addr(addr):
     """If addr is IPv6 then return addr wrapped between '[' and ']',
     otherwise return it intact."""
-    if _is_ipv6(addr):
+    if ipaddr.is_ipv6(addr):
         return "".join(("[", addr, "]", ))
     else:
         return addr
@@ -145,57 +118,59 @@ class Remote(object):
         self.log.debug("Remote mkdir: %s", path)
         self.run(["mkdir", "-p", path])
 
-    def transfer(self, source, destination):
+    def transfer(self, source, destination, jobs=None):
         """Transfer a file or directory to the remote side."""
         self.log.debug("Remote transfer: %s -> %s", source, destination)
         self.rmdir(destination)
         if os.path.isdir(source):
             self.mkdir(destination)
-            for f in glob.glob1(source, '*'):
+            if jobs:
-                command = self._copy.split()
+                self._transfer_dir_parallel(source, destination, jobs)
-                path = os.path.join(source, f)
+            else:
-                command.extend([path, '{0}:{1}'.format(
+                self._transfer_dir_sequential(source, destination)
-                    _wrap_addr(self.target_host[0]), destination)])
+        elif jobs:
-                self._run_command(command)
+            raise cdist.Error("Source {} is not a directory".format(source))
         else:
             command = self._copy.split()
             command.extend([source, '{0}:{1}'.format(
                 _wrap_addr(self.target_host[0]), destination)])
             self._run_command(command)
 
-    def transfer_dir_parallel(self, source, destination, jobs):
+    def _transfer_dir_sequential(self, source, destination):
-        """Transfer a directory to the remote side in parallel mode."""
+        for f in glob.glob1(source, '*'):
-        self.log.debug("Remote transfer: %s -> %s", source, destination)
+            command = self._copy.split()
-        self.rmdir(destination)
+            path = os.path.join(source, f)
-        if os.path.isdir(source):
+            command.extend([path, '{0}:{1}'.format(
-            self.mkdir(destination)
+                _wrap_addr(self.target_host[0]), destination)])
-            self.log.info("Remote transfer in {} parallel jobs".format(
+            self._run_command(command)
-                jobs))
-            self.log.debug("Multiprocessing start method is {}".format(
-                multiprocessing.get_start_method()))
-            self.log.debug(("Starting multiprocessing Pool for parallel "
-                           "remote transfer"))
-            with multiprocessing.Pool(jobs) as pool:
-                self.log.debug("Starting async for parallel transfer")
-                commands = []
-                for f in glob.glob1(source, '*'):
-                    command = self._copy.split()
-                    path = os.path.join(source, f)
-                    command.extend([path, '{0}:{1}'.format(
-                        _wrap_addr(self.target_host[0]), destination)])
-                    commands.append(command)
-                results = [
-                    pool.apply_async(self._run_command, (cmd,))
-                    for cmd in commands
-                ]
 
-                self.log.debug("Waiting async results for parallel transfer")
+    def _transfer_dir_parallel(self, source, destination, jobs):
-                for r in results:
+        """Transfer a directory to the remote side in parallel mode."""
-                    r.get()  # self._run_command returns None
+        self.log.info("Remote transfer in {} parallel jobs".format(
-                self.log.debug(("Multiprocessing for parallel transfer "
+            jobs))
-                               "finished"))
+        self.log.debug("Multiprocessing start method is {}".format(
-        else:
+            multiprocessing.get_start_method()))
-            raise cdist.Error("Source {} is not a directory".format(source))
+        self.log.debug(("Starting multiprocessing Pool for parallel "
+                        "remote transfer"))
+        with multiprocessing.Pool(jobs) as pool:
+            self.log.debug("Starting async for parallel transfer")
+            commands = []
+            for f in glob.glob1(source, '*'):
+                command = self._copy.split()
+                path = os.path.join(source, f)
+                command.extend([path, '{0}:{1}'.format(
+                    _wrap_addr(self.target_host[0]), destination)])
+                commands.append(command)
+            results = [
+                pool.apply_async(self._run_command, (cmd,))
+                for cmd in commands
+            ]
+
+            self.log.debug("Waiting async results for parallel transfer")
+            for r in results:
+                r.get()  # self._run_command returns None
+            self.log.debug(("Multiprocessing for parallel transfer "
+                            "finished"))
 
     def run_script(self, script, env=None, return_output=False):
         """Run the given script with the given environment on the remote side.

cdist/hostsource.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import fileinput
+
+
+class HostSource(object):
+    """
+    Host source object.
+    Source can be a sequence or filename (stdin if \'-\').
+    In case of filename each line represents one host.
+    """
+    def __init__(self, source):
+        self.source = source
+
+    def _process_file_line(self, line):
+        """Return host from read line or None if no host present."""
+        if not line:
+            return None
+        # remove comment if present
+        comment_index = line.find('#')
+        if comment_index >= 0:
+            host = line[:comment_index]
+        else:
+            host = line
+        # remove leading and trailing whitespaces
+        host = host.strip()
+        # skip empty lines
+        if host:
+            return host
+        else:
+            return None
+
+    def _hosts_from_sequence(self):
+        for host in self.source:
+            yield host
+
+    def _hosts_from_file(self):
+        for line in fileinput.input(files=(self.source)):
+            host = self._process_file_line(line)
+            if host:
+                yield host
+
+    def hosts(self):
+        if not self.source:
+            return
+
+        if isinstance(self.source, str):
+            yield from self._hosts_from_file()
+        else:
+            yield from self._hosts_from_sequence()
+
+    def __call__(self):
+        yield from self.hosts()

cdist/test/exec/remote.py

@@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase):
             source_file_name = os.path.split(source_file)[-1]
             filenames.append(source_file_name)
         target = self.mkdtemp(dir=self.temp_dir)
-        self.remote.transfer_dir_parallel(source, target,
+        self.remote.transfer(source, target,
-                                          multiprocessing.cpu_count())
+                             multiprocessing.cpu_count())
         # test if the payload files are in the target directory
         for filename in filenames:
             self.assertTrue(os.path.isfile(os.path.join(target, filename)))

cdist/util/ipaddr.py

@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import socket
+import logging
+
+
+def resolve_target_addresses(host):
+    log = logging.getLogger(host)
+    try:
+        # getaddrinfo returns a list of 5-tuples:
+        # (family, type, proto, canonname, sockaddr)
+        # where sockaddr is:
+        # (address, port) for AF_INET,
+        # (address, port, flow_info, scopeid) for AF_INET6
+        ip_addr = socket.getaddrinfo(
+                host, None, type=socket.SOCK_STREAM)[0][4][0]
+        # gethostbyaddr returns triple
+        # (hostname, aliaslist, ipaddrlist)
+        host_name = socket.gethostbyaddr(ip_addr)[0]
+        log.debug("derived host_name for host \"{}\": {}".format(
+            host, host_name))
+    except (socket.gaierror, socket.herror) as e:
+        log.warn("Could not derive host_name for {}"
+                 ", $host_name will be empty. Error is: {}".format(host, e))
+        # in case of error provide empty value
+        host_name = ''
+
+    try:
+        host_fqdn = socket.getfqdn(host)
+        log.debug("derived host_fqdn for host \"{}\": {}".format(
+            host, host_fqdn))
+    except socket.herror as e:
+        log.warn("Could not derive host_fqdn for {}"
+                 ", $host_fqdn will be empty. Error is: {}".format(host, e))
+        # in case of error provide empty value
+        host_fqdn = ''
+
+    return (host, host_name, host_fqdn)
+
+
+# check whether addr is IPv6
+try:
+    # python 3.3+
+    import ipaddress
+
+    def is_ipv6(addr):
+        try:
+            return ipaddress.ip_address(addr).version == 6
+        except ValueError:
+            return False
+except ImportError:
+    # fallback for older python versions
+    def is_ipv6(addr):
+        try:
+            socket.inet_aton(addr)
+            return False
+        except socket.error:
+            pass
+        try:
+            socket.inet_pton(socket.AF_INET6, addr)
+            return True
+        except socket.error:
+            pass
+        return False

cdist/util/remoteutil.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+def inspect_ssh_mux_opts():
+    """Inspect whether or not ssh supports multiplexing options.
+
+       Return string containing multiplexing options if supported.
+       If ControlPath is supported then placeholder for that path is
+       specified and can be used for final string formatting.
+       For example, this function can return string:
+       "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
+       Then it can be formatted:
+       mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
+    """
+    import subprocess
+
+    wanted_mux_opts = {
+        "ControlPath": "{}",
+        "ControlMaster": "auto",
+        "ControlPersist": "125",
+    }
+    mux_opts = " ".join([" -o {}={}".format(
+        x, wanted_mux_opts[x]) for x in wanted_mux_opts])
+    try:
+        subprocess.check_output("ssh {}".format(mux_opts),
+                                stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        subproc_output = e.output.decode().lower()
+        if "bad configuration option" in subproc_output:
+            return ""
+    return mux_opts

docs/src/cdist-reference.rst.sh

@@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC
 
 CDIST_REMOTE_COPY
     Use this command for remote copy (should behave like scp).
+
+CDIST_BETA
+    Enable beta functionalities.
 eof

docs/src/man1/cdist.rst

@@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC
 CDIST_REMOTE_COPY
     Use this command for remote copy (should behave like scp).
 
+CDIST_BETA
+    Enable beta functionalities.
+
 EXIT STATUS
 -----------
 The following exit values shall be returned:

scripts/cdist

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 # -*- coding: utf-8 -*-
 #
-# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
 # 2016 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
@@ -24,182 +24,25 @@
 import collections
 import logging
 
-# list of beta sub-commands
-BETA_COMMANDS = ['install', ]
-# list of beta arguments for sub-commands
-BETA_ARGS = {
-    'config': ['jobs', ],
-}
-
-
-def check_positive_int(value):
-    import argparse
-
-    try:
-        val = int(value)
-    except ValueError as e:
-        raise argparse.ArgumentTypeError(
-                "{} is invalid int value".format(value))
-    if val <= 0:
-        raise argparse.ArgumentTypeError(
-                "{} is invalid positive int value".format(val))
-    return val
-
-
-def check_beta(args_dict):
-    if 'beta' not in args_dict:
-        args_dict['beta'] = False
-    # Check only if beta is not enabled: if beta option is specified then
-    # raise error.
-    if not args_dict['beta']:
-        cmd = args_dict['command']
-        # first check if command is beta
-        if cmd in BETA_COMMANDS:
-            raise cdist.CdistBetaRequired(cmd)
-        # then check if command's argument is beta
-        if cmd in BETA_ARGS:
-            for arg in BETA_ARGS[cmd]:
-                if arg in args_dict and args_dict[arg]:
-                    raise cdist.CdistBetaRequired(cmd, arg)
-
-
-_verbosity_level = {
-    0: logging.ERROR,
-    1: logging.WARNING,
-    2: logging.INFO,
-}
-_verbosity_level = collections.defaultdict(
-    lambda: logging.DEBUG, _verbosity_level)
-
 
 def commandline():
     """Parse command line"""
-    import argparse
 
+    import cdist.argparse
     import cdist.banner
     import cdist.config
     import cdist.install
     import cdist.shell
     import shutil
     import os
-    import multiprocessing
-
-    # Construct parser others can reuse
-    parser = {}
-    # Options _all_ parsers have in common
-    parser['loglevel'] = argparse.ArgumentParser(add_help=False)
-    parser['loglevel'].add_argument(
-            '-d', '--debug',
-            help=('Set log level to debug (deprecated, use -vvv instead)'),
-            action='store_true', default=False)
-    parser['loglevel'].add_argument(
-            '-v', '--verbose',
-            help=('Increase log level, be more verbose. Use it more than once '
-                  'to increase log level. The order of levels from the lowest '
-                  'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
-            action='count', default=0)
-
-    # Main subcommand parser
-    parser['main'] = argparse.ArgumentParser(
-            description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
-    parser['main'].add_argument(
-            '-V', '--version', help='Show version', action='version',
-            version='%(prog)s ' + cdist.VERSION)
-    parser['sub'] = parser['main'].add_subparsers(
-            title="Commands", dest="command")
-
-    # Banner
-    parser['banner'] = parser['sub'].add_parser(
-            'banner', parents=[parser['loglevel']])
-    parser['banner'].set_defaults(func=cdist.banner.banner)
-
-    # Config
-    parser['config'] = parser['sub'].add_parser(
-            'config', parents=[parser['loglevel']])
-    parser['config'].add_argument(
-            'host', nargs='*', help='host(s) to operate on')
-    parser['config'].add_argument(
-           '-b', '--enable-beta',
-           help=('Enable beta functionalities. Beta functionalities '
-                 'include the following options: -j/--jobs.'),
-           action='store_true', dest='beta', default=False)
-    parser['config'].add_argument(
-            '-c', '--conf-dir',
-            help=('Add configuration directory (can be repeated, '
-                  'last one wins)'), action='append')
-    parser['config'].add_argument(
-            '-f', '--file',
-            help=('Read additional hosts to operate on from specified file '
-                  'or from stdin if \'-\' (each host on separate line). '
-                  'If no host or host file is specified then, by default, '
-                  'read hosts from stdin.'),
-            dest='hostfile', required=False)
-    parser['config'].add_argument(
-           '-i', '--initial-manifest',
-           help='Path to a cdist manifest or \'-\' to read from stdin.',
-           dest='manifest', required=False)
-    parser['config'].add_argument(
-           '-j', '--jobs', nargs='?', type=check_positive_int,
-           help=('Specify the maximum number of parallel jobs, currently '
-                 'only global explorers are supported (currently in beta'),
-           action='store', dest='jobs',
-           const=multiprocessing.cpu_count())
-    parser['config'].add_argument(
-           '-n', '--dry-run',
-           help='Do not execute code', action='store_true')
-    parser['config'].add_argument(
-           '-o', '--out-dir',
-           help='Directory to save cdist output in', dest="out_path")
-    parser['config'].add_argument(
-           '-p', '--parallel',
-           help='Operate on multiple hosts in parallel',
-           action='store_true', dest='parallel')
-    parser['config'].add_argument(
-           '-s', '--sequential',
-           help='Operate on multiple hosts sequentially (default)',
-           action='store_false', dest='parallel')
-    # remote-copy and remote-exec defaults are environment variables
-    # if set; if not then None - these will be futher handled after
-    # parsing to determine implementation default
-    parser['config'].add_argument(
-           '--remote-copy',
-           help='Command to use for remote copy (should behave like scp)',
-           action='store', dest='remote_copy',
-           default=os.environ.get('CDIST_REMOTE_COPY'))
-    parser['config'].add_argument(
-           '--remote-exec',
-           help=('Command to use for remote execution '
-                 '(should behave like ssh)'),
-           action='store', dest='remote_exec',
-           default=os.environ.get('CDIST_REMOTE_EXEC'))
-    parser['config'].set_defaults(func=cdist.config.Config.commandline)
-
-    # Install
-    parser['install'] = parser['sub'].add_parser('install', add_help=False,
-                                                 parents=[parser['config']])
-    parser['install'].set_defaults(func=cdist.install.Install.commandline)
-
-    # Shell
-    parser['shell'] = parser['sub'].add_parser(
-            'shell', parents=[parser['loglevel']])
-    parser['shell'].add_argument(
-            '-s', '--shell',
-            help=('Select shell to use, defaults to current shell. Used shell'
-                  ' should be POSIX compatible shell.'))
-    parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
-
-    for p in parser:
-        parser[p].epilog = (
-                "Get cdist at http://www.nico.schottelius.org/software/cdist/")
 
+    parser = cdist.argparse.get_parsers()
     args = parser['main'].parse_args(sys.argv[1:])
 
     # Loglevels are handled globally in here
-    if args.debug:
+    retval = cdist.argparse.handle_loglevel(args)
-        log.warning("-d/--debug is deprecated, use -vvv instead")
+    if retval:
-        args.verbose = 3
+        log.warning(retval)
-
-    logging.root.setLevel(_verbosity_level[args.verbose])
 
     log.debug(args)
     log.info("version %s" % cdist.VERSION)
@@ -219,17 +62,16 @@ def commandline():
         parser['main'].print_help()
         sys.exit(0)
 
-    check_beta(vars(args))
+    cdist.argparse.check_beta(vars(args))
     args.func(args)
 
 if __name__ == "__main__":
-    # Sys is needed for sys.exit()
     import sys
 
     cdistpythonversion = '3.2'
     if sys.version < cdistpythonversion:
-        print('Python >= ' + cdistpythonversion +
+        print('Python >= {} is required on the source host.'.format(
-              ' is required on the source host.', file=sys.stderr)
+                cdistpythonversion), file=sys.stderr)
         sys.exit(1)
 
     exit_code = 0