diff --git a/cdist/__init__.py b/cdist/__init__.py
index b6f5c8cb..c142230c 100644
--- a/cdist/__init__.py
+++ b/cdist/__init__.py
@@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error):
err_msg = ("\'{}\' command is beta, but beta is "
"not enabled. If you want to use it please enable beta "
"functionalities by using the -b/--enable-beta command "
- "line flag.")
+ "line flag or setting CDIST_BETA env var.")
fmt_args = [self.command, ]
else:
err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta "
"is not enabled. If you want to use it please enable "
"beta functionalities by using the -b/--enable-beta "
- "command line flag.")
+ "command line flag or setting CDIST_BETA env var.")
fmt_args = [self.arg, self.command, ]
return err_msg.format(*fmt_args)
diff --git a/cdist/argparse.py b/cdist/argparse.py
new file mode 100644
index 00000000..04f6e6a4
--- /dev/null
+++ b/cdist/argparse.py
@@ -0,0 +1,208 @@
+import argparse
+import cdist
+import multiprocessing
+import os
+import logging
+import collections
+
+
+# set of beta sub-commands
+BETA_COMMANDS = set(('install', ))
+# set of beta arguments for sub-commands
+BETA_ARGS = {
+ 'config': set(('jobs', )),
+}
+EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
+# Parser others can reuse
+parser = None
+
+
+_verbosity_level = {
+ 0: logging.ERROR,
+ 1: logging.WARNING,
+ 2: logging.INFO,
+}
+_verbosity_level = collections.defaultdict(
+ lambda: logging.DEBUG, _verbosity_level)
+
+
+def add_beta_command(cmd):
+ BETA_COMMANDS.add(cmd)
+
+
+def add_beta_arg(cmd, arg):
+ if cmd in BETA_ARGS:
+ if arg not in BETA_ARGS[cmd]:
+ BETA_ARGS[cmd].append(arg)
+ else:
+ BETA_ARGS[cmd] = set((arg, ))
+
+
+def check_beta(args_dict):
+ if 'beta' not in args_dict:
+ args_dict['beta'] = False
+ # Check only if beta is not enabled: if beta option is specified then
+ # raise error.
+ if not args_dict['beta']:
+ cmd = args_dict['command']
+ # first check if command is beta
+ if cmd in BETA_COMMANDS:
+ raise cdist.CdistBetaRequired(cmd)
+ # then check if some command's argument is beta
+ if cmd in BETA_ARGS:
+ for arg in BETA_ARGS[cmd]:
+ if arg in args_dict and args_dict[arg]:
+ raise cdist.CdistBetaRequired(cmd, arg)
+
+
+def check_positive_int(value):
+ import argparse
+
+ try:
+ val = int(value)
+ except ValueError:
+ raise argparse.ArgumentTypeError(
+ "{} is invalid int value".format(value))
+ if val <= 0:
+ raise argparse.ArgumentTypeError(
+ "{} is invalid positive int value".format(val))
+ return val
+
+
+def get_parsers():
+ global parser
+
+ # Construct parser others can reuse
+ if parser:
+ return parser
+ else:
+ parser = {}
+ # Options _all_ parsers have in common
+ parser['loglevel'] = argparse.ArgumentParser(add_help=False)
+ parser['loglevel'].add_argument(
+ '-d', '--debug',
+ help=('Set log level to debug (deprecated, use -vvv instead)'),
+ action='store_true', default=False)
+ parser['loglevel'].add_argument(
+ '-v', '--verbose',
+ help=('Increase log level, be more verbose. Use it more than once '
+ 'to increase log level. The order of levels from the lowest '
+ 'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
+ action='count', default=0)
+
+ parser['beta'] = argparse.ArgumentParser(add_help=False)
+ parser['beta'].add_argument(
+ '-b', '--beta',
+ help=('Enable beta functionalities. '
+ 'Can also be enabled using CDIST_BETA env var.'),
+ action='store_true', dest='beta',
+ default='CDIST_BETA' in os.environ)
+
+ # Main subcommand parser
+ parser['main'] = argparse.ArgumentParser(
+ description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
+ parser['main'].add_argument(
+ '-V', '--version', help='Show version', action='version',
+ version='%(prog)s ' + cdist.VERSION)
+ parser['sub'] = parser['main'].add_subparsers(
+ title="Commands", dest="command")
+
+ # Banner
+ parser['banner'] = parser['sub'].add_parser(
+ 'banner', parents=[parser['loglevel']])
+ parser['banner'].set_defaults(func=cdist.banner.banner)
+
+ # Config
+ parser['config_main'] = argparse.ArgumentParser(add_help=False)
+ parser['config_main'].add_argument(
+ '-c', '--conf-dir',
+ help=('Add configuration directory (can be repeated, '
+ 'last one wins)'), action='append')
+ parser['config_main'].add_argument(
+ '-i', '--initial-manifest',
+ help='path to a cdist manifest or \'-\' to read from stdin.',
+ dest='manifest', required=False)
+ parser['config_main'].add_argument(
+ '-j', '--jobs', nargs='?',
+ type=check_positive_int,
+ help=('Specify the maximum number of parallel jobs, currently '
+ 'only global explorers are supported'),
+ action='store', dest='jobs',
+ const=multiprocessing.cpu_count())
+ parser['config_main'].add_argument(
+ '-n', '--dry-run',
+ help='do not execute code', action='store_true')
+ parser['config_main'].add_argument(
+ '-o', '--out-dir',
+ help='directory to save cdist output in', dest="out_path")
+
+ # remote-copy and remote-exec defaults are environment variables
+ # if set; if not then None - these will be futher handled after
+ # parsing to determine implementation default
+ parser['config_main'].add_argument(
+ '--remote-copy',
+ help='Command to use for remote copy (should behave like scp)',
+ action='store', dest='remote_copy',
+ default=os.environ.get('CDIST_REMOTE_COPY'))
+ parser['config_main'].add_argument(
+ '--remote-exec',
+ help=('Command to use for remote execution '
+ '(should behave like ssh)'),
+ action='store', dest='remote_exec',
+ default=os.environ.get('CDIST_REMOTE_EXEC'))
+
+ # Config
+ parser['config_args'] = argparse.ArgumentParser(add_help=False)
+ parser['config_args'].add_argument(
+ 'host', nargs='*', help='host(s) to operate on')
+ parser['config_args'].add_argument(
+ '-f', '--file',
+ help=('Read additional hosts to operate on from specified file '
+ 'or from stdin if \'-\' (each host on separate line). '
+ 'If no host or host file is specified then, by default, '
+ 'read hosts from stdin.'),
+ dest='hostfile', required=False)
+ parser['config_args'].add_argument(
+ '-p', '--parallel',
+ help='operate on multiple hosts in parallel',
+ action='store_true', dest='parallel')
+ parser['config_args'].add_argument(
+ '-s', '--sequential',
+ help='operate on multiple hosts sequentially (default)',
+ action='store_false', dest='parallel')
+ parser['config'] = parser['sub'].add_parser(
+ 'config', parents=[parser['loglevel'], parser['beta'],
+ parser['config_main'],
+ parser['config_args']])
+ parser['config'].set_defaults(func=cdist.config.Config.commandline)
+
+ # Install
+ parser['install'] = parser['sub'].add_parser('install', add_help=False,
+ parents=[parser['config']])
+ parser['install'].set_defaults(func=cdist.install.Install.commandline)
+
+ # Shell
+ parser['shell'] = parser['sub'].add_parser(
+ 'shell', parents=[parser['loglevel']])
+ parser['shell'].add_argument(
+ '-s', '--shell',
+ help=('Select shell to use, defaults to current shell. Used shell'
+ ' should be POSIX compatible shell.'))
+ parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
+
+ for p in parser:
+ parser[p].epilog = EPILOG
+
+ return parser
+
+
+def handle_loglevel(args):
+ if args.debug:
+ retval = "-d/--debug is deprecated, use -vvv instead"
+ args.verbose = 3
+ else:
+ retval = None
+
+ logging.root.setLevel(_verbosity_level[args.verbose])
+
+ return retval
diff --git a/cdist/config.py b/cdist/config.py
index b8d0672c..6b57e7bf 100644
--- a/cdist/config.py
+++ b/cdist/config.py
@@ -22,50 +22,21 @@
import logging
import os
-import shutil
import sys
import time
-import pprint
import itertools
import tempfile
import socket
import cdist
+import cdist.hostsource
import cdist.exec.local
import cdist.exec.remote
+import cdist.util.ipaddr as ipaddr
from cdist import core
-
-
-def inspect_ssh_mux_opts():
- """Inspect whether or not ssh supports multiplexing options.
-
- Return string containing multiplexing options if supported.
- If ControlPath is supported then placeholder for that path is
- specified and can be used for final string formatting.
- For example, this function can return string:
- "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
- Then it can be formatted:
- mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
- """
- import subprocess
-
- wanted_mux_opts = {
- "ControlPath": "{}",
- "ControlMaster": "auto",
- "ControlPersist": "125",
- }
- mux_opts = " ".join([" -o {}={}".format(
- x, wanted_mux_opts[x]) for x in wanted_mux_opts])
- try:
- subprocess.check_output("ssh {}".format(mux_opts),
- stderr=subprocess.STDOUT, shell=True)
- except subprocess.CalledProcessError as e:
- subproc_output = e.output.decode().lower()
- if "bad configuration option" in subproc_output:
- return ""
- return mux_opts
+from cdist.util.remoteutil import inspect_ssh_mux_opts
class Config(object):
@@ -89,55 +60,17 @@ class Config(object):
self.local.create_files_dirs()
self.remote.create_files_dirs()
- @staticmethod
- def hostfile_process_line(line):
- """Return host from read line or None if no host present."""
- if not line:
- return None
- # remove comment if present
- comment_index = line.find('#')
- if comment_index >= 0:
- host = line[:comment_index]
- else:
- host = line
- # remove leading and trailing whitespaces
- host = host.strip()
- # skip empty lines
- if host:
- return host
- else:
- return None
-
@staticmethod
def hosts(source):
- """Yield hosts from source.
- Source can be a sequence or filename (stdin if \'-\').
- In case of filename each line represents one host.
- """
- if isinstance(source, str):
- import fileinput
- try:
- for host in fileinput.input(files=(source)):
- host = Config.hostfile_process_line(host)
- if host:
- yield host
- except (IOError, OSError, UnicodeError) as e:
- raise cdist.Error(
- "Error reading hosts from file \'{}\': {}".format(
- source, e))
- else:
- if source:
- for host in source:
- yield host
+ try:
+ yield from cdist.hostsource.HostSource(source)()
+ except (IOError, OSError, UnicodeError) as e:
+ raise cdist.Error(
+ "Error reading hosts from \'{}\': {}".format(
+ source, e))
@classmethod
- def commandline(cls, args):
- """Configure remote system"""
- import multiprocessing
-
- # FIXME: Refactor relict - remove later
- log = logging.getLogger("cdist")
-
+ def _check_and_prepare_args(cls, args):
if args.manifest == '-' and args.hostfile == '-':
raise cdist.Error(("Cannot read both, manifest and host file, "
"from stdin"))
@@ -162,10 +95,6 @@ class Config(object):
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
- process = {}
- failed_hosts = []
- time_start = time.time()
-
# default remote cmd patterns
args.remote_exec_pattern = None
args.remote_copy_pattern = None
@@ -182,10 +111,29 @@ class Config(object):
if args_dict['remote_copy'] is None:
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
+ @classmethod
+ def _base_root_path(cls, args):
if args.out_path:
base_root_path = args.out_path
else:
base_root_path = tempfile.mkdtemp()
+ return base_root_path
+
+ @classmethod
+ def commandline(cls, args):
+ """Configure remote system"""
+ import multiprocessing
+
+ # FIXME: Refactor relict - remove later
+ log = logging.getLogger("cdist")
+
+ cls._check_and_prepare_args(args)
+
+ process = {}
+ failed_hosts = []
+ time_start = time.time()
+
+ base_root_path = cls._base_root_path(args)
hostcnt = 0
for host in itertools.chain(cls.hosts(args.host),
@@ -227,6 +175,24 @@ class Config(object):
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
+ @classmethod
+ def _resolve_remote_cmds(cls, args, host_base_path):
+ control_path = os.path.join(host_base_path, "ssh-control-path")
+ # If we constructed patterns for remote commands then there is
+ # placeholder for ssh ControlPath, format it and we have unique
+ # ControlPath for each host.
+ #
+ # If not then use args.remote_exec/copy that user specified.
+ if args.remote_exec_pattern:
+ remote_exec = args.remote_exec_pattern.format(control_path)
+ else:
+ remote_exec = args.remote_exec
+ if args.remote_copy_pattern:
+ remote_copy = args.remote_copy_pattern.format(control_path)
+ else:
+ remote_copy = args.remote_copy
+ return (remote_exec, remote_copy, )
+
@classmethod
def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
"""Configure ONE system"""
@@ -234,57 +200,14 @@ class Config(object):
log = logging.getLogger(host)
try:
- control_path = os.path.join(host_base_path, "ssh-control-path")
- # If we constructed patterns for remote commands then there is
- # placeholder for ssh ControlPath, format it and we have unique
- # ControlPath for each host.
- #
- # If not then use args.remote_exec/copy that user specified.
- if args.remote_exec_pattern:
- remote_exec = args.remote_exec_pattern.format(control_path)
- else:
- remote_exec = args.remote_exec
- if args.remote_copy_pattern:
- remote_copy = args.remote_copy_pattern.format(control_path)
- else:
- remote_copy = args.remote_copy
+ remote_exec, remote_copy = cls._resolve_remote_cmds(
+ args, host_base_path)
log.debug("remote_exec for host \"{}\": {}".format(
host, remote_exec))
log.debug("remote_copy for host \"{}\": {}".format(
host, remote_copy))
- try:
- # getaddrinfo returns a list of 5-tuples:
- # (family, type, proto, canonname, sockaddr)
- # where sockaddr is:
- # (address, port) for AF_INET,
- # (address, port, flow_info, scopeid) for AF_INET6
- ip_addr = socket.getaddrinfo(
- host, None, type=socket.SOCK_STREAM)[0][4][0]
- # gethostbyaddr returns triple
- # (hostname, aliaslist, ipaddrlist)
- host_name = socket.gethostbyaddr(ip_addr)[0]
- log.debug("derived host_name for host \"{}\": {}".format(
- host, host_name))
- except (socket.gaierror, socket.herror) as e:
- log.warn("Could not derive host_name for {}"
- ", $host_name will be empty. Error is: {}".format(
- host, e))
- # in case of error provide empty value
- host_name = ''
-
- try:
- host_fqdn = socket.getfqdn(host)
- log.debug("derived host_fqdn for host \"{}\": {}".format(
- host, host_fqdn))
- except socket.herror as e:
- log.warn("Could not derive host_fqdn for {}"
- ", $host_fqdn will be empty. Error is: {}".format(
- host, e))
- # in case of error provide empty value
- host_fqdn = ''
-
- target_host = (host, host_name, host_fqdn)
+ target_host = ipaddr.resolve_target_addresses(host)
log.debug("target_host: {}".format(target_host))
local = cdist.exec.local.Local(
diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py
index ef85431c..23996240 100644
--- a/cdist/core/explorer.py
+++ b/cdist/core/explorer.py
@@ -149,14 +149,9 @@ class Explorer(object):
def transfer_global_explorers(self):
"""Transfer the global explorers to the remote side."""
self.remote.mkdir(self.remote.global_explorer_path)
- if self.jobs is None:
- self.remote.transfer(self.local.global_explorer_path,
- self.remote.global_explorer_path)
- else:
- self.remote.transfer_dir_parallel(
- self.local.global_explorer_path,
- self.remote.global_explorer_path,
- self.jobs)
+ self.remote.transfer(self.local.global_explorer_path,
+ self.remote.global_explorer_path,
+ self.jobs)
self.remote.run(["chmod", "0700",
"%s/*" % (self.remote.global_explorer_path)])
diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py
index f374262f..440aafa7 100644
--- a/cdist/exec/remote.py
+++ b/cdist/exec/remote.py
@@ -30,40 +30,13 @@ import multiprocessing
import cdist
import cdist.exec.util as exec_util
-
-
-# check whether addr is IPv6
-try:
- # python 3.3+
- import ipaddress
-
- def _is_ipv6(addr):
- try:
- return ipaddress.ip_address(addr).version == 6
- except ValueError:
- return False
-except ImportError:
- # fallback for older python versions
- import socket
-
- def _is_ipv6(addr):
- try:
- socket.inet_aton(addr)
- return False
- except socket.error:
- pass
- try:
- socket.inet_pton(socket.AF_INET6, addr)
- return True
- except socket.error:
- pass
- return False
+import cdist.util.ipaddr as ipaddr
def _wrap_addr(addr):
"""If addr is IPv6 then return addr wrapped between '[' and ']',
otherwise return it intact."""
- if _is_ipv6(addr):
+ if ipaddr.is_ipv6(addr):
return "".join(("[", addr, "]", ))
else:
return addr
@@ -145,57 +118,59 @@ class Remote(object):
self.log.debug("Remote mkdir: %s", path)
self.run(["mkdir", "-p", path])
- def transfer(self, source, destination):
+ def transfer(self, source, destination, jobs=None):
"""Transfer a file or directory to the remote side."""
self.log.debug("Remote transfer: %s -> %s", source, destination)
self.rmdir(destination)
if os.path.isdir(source):
self.mkdir(destination)
- for f in glob.glob1(source, '*'):
- command = self._copy.split()
- path = os.path.join(source, f)
- command.extend([path, '{0}:{1}'.format(
- _wrap_addr(self.target_host[0]), destination)])
- self._run_command(command)
+ if jobs:
+ self._transfer_dir_parallel(source, destination, jobs)
+ else:
+ self._transfer_dir_sequential(source, destination)
+ elif jobs:
+ raise cdist.Error("Source {} is not a directory".format(source))
else:
command = self._copy.split()
command.extend([source, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
self._run_command(command)
- def transfer_dir_parallel(self, source, destination, jobs):
- """Transfer a directory to the remote side in parallel mode."""
- self.log.debug("Remote transfer: %s -> %s", source, destination)
- self.rmdir(destination)
- if os.path.isdir(source):
- self.mkdir(destination)
- self.log.info("Remote transfer in {} parallel jobs".format(
- jobs))
- self.log.debug("Multiprocessing start method is {}".format(
- multiprocessing.get_start_method()))
- self.log.debug(("Starting multiprocessing Pool for parallel "
- "remote transfer"))
- with multiprocessing.Pool(jobs) as pool:
- self.log.debug("Starting async for parallel transfer")
- commands = []
- for f in glob.glob1(source, '*'):
- command = self._copy.split()
- path = os.path.join(source, f)
- command.extend([path, '{0}:{1}'.format(
- _wrap_addr(self.target_host[0]), destination)])
- commands.append(command)
- results = [
- pool.apply_async(self._run_command, (cmd,))
- for cmd in commands
- ]
+ def _transfer_dir_sequential(self, source, destination):
+ for f in glob.glob1(source, '*'):
+ command = self._copy.split()
+ path = os.path.join(source, f)
+ command.extend([path, '{0}:{1}'.format(
+ _wrap_addr(self.target_host[0]), destination)])
+ self._run_command(command)
- self.log.debug("Waiting async results for parallel transfer")
- for r in results:
- r.get() # self._run_command returns None
- self.log.debug(("Multiprocessing for parallel transfer "
- "finished"))
- else:
- raise cdist.Error("Source {} is not a directory".format(source))
+ def _transfer_dir_parallel(self, source, destination, jobs):
+ """Transfer a directory to the remote side in parallel mode."""
+ self.log.info("Remote transfer in {} parallel jobs".format(
+ jobs))
+ self.log.debug("Multiprocessing start method is {}".format(
+ multiprocessing.get_start_method()))
+ self.log.debug(("Starting multiprocessing Pool for parallel "
+ "remote transfer"))
+ with multiprocessing.Pool(jobs) as pool:
+ self.log.debug("Starting async for parallel transfer")
+ commands = []
+ for f in glob.glob1(source, '*'):
+ command = self._copy.split()
+ path = os.path.join(source, f)
+ command.extend([path, '{0}:{1}'.format(
+ _wrap_addr(self.target_host[0]), destination)])
+ commands.append(command)
+ results = [
+ pool.apply_async(self._run_command, (cmd,))
+ for cmd in commands
+ ]
+
+ self.log.debug("Waiting async results for parallel transfer")
+ for r in results:
+ r.get() # self._run_command returns None
+ self.log.debug(("Multiprocessing for parallel transfer "
+ "finished"))
def run_script(self, script, env=None, return_output=False):
"""Run the given script with the given environment on the remote side.
diff --git a/cdist/hostsource.py b/cdist/hostsource.py
new file mode 100644
index 00000000..9c2c0616
--- /dev/null
+++ b/cdist/hostsource.py
@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see .
+#
+#
+
+import fileinput
+
+
+class HostSource(object):
+ """
+ Host source object.
+ Source can be a sequence or filename (stdin if \'-\').
+ In case of filename each line represents one host.
+ """
+ def __init__(self, source):
+ self.source = source
+
+ def _process_file_line(self, line):
+ """Return host from read line or None if no host present."""
+ if not line:
+ return None
+ # remove comment if present
+ comment_index = line.find('#')
+ if comment_index >= 0:
+ host = line[:comment_index]
+ else:
+ host = line
+ # remove leading and trailing whitespaces
+ host = host.strip()
+ # skip empty lines
+ if host:
+ return host
+ else:
+ return None
+
+ def _hosts_from_sequence(self):
+ for host in self.source:
+ yield host
+
+ def _hosts_from_file(self):
+ for line in fileinput.input(files=(self.source)):
+ host = self._process_file_line(line)
+ if host:
+ yield host
+
+ def hosts(self):
+ if not self.source:
+ return
+
+ if isinstance(self.source, str):
+ yield from self._hosts_from_file()
+ else:
+ yield from self._hosts_from_sequence()
+
+ def __call__(self):
+ yield from self.hosts()
diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py
index 45dabb18..371d17e3 100644
--- a/cdist/test/exec/remote.py
+++ b/cdist/test/exec/remote.py
@@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase):
source_file_name = os.path.split(source_file)[-1]
filenames.append(source_file_name)
target = self.mkdtemp(dir=self.temp_dir)
- self.remote.transfer_dir_parallel(source, target,
- multiprocessing.cpu_count())
+ self.remote.transfer(source, target,
+ multiprocessing.cpu_count())
# test if the payload files are in the target directory
for filename in filenames:
self.assertTrue(os.path.isfile(os.path.join(target, filename)))
diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py
new file mode 100644
index 00000000..71477682
--- /dev/null
+++ b/cdist/util/ipaddr.py
@@ -0,0 +1,83 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see .
+#
+#
+
+import socket
+import logging
+
+
+def resolve_target_addresses(host):
+ log = logging.getLogger(host)
+ try:
+ # getaddrinfo returns a list of 5-tuples:
+ # (family, type, proto, canonname, sockaddr)
+ # where sockaddr is:
+ # (address, port) for AF_INET,
+ # (address, port, flow_info, scopeid) for AF_INET6
+ ip_addr = socket.getaddrinfo(
+ host, None, type=socket.SOCK_STREAM)[0][4][0]
+ # gethostbyaddr returns triple
+ # (hostname, aliaslist, ipaddrlist)
+ host_name = socket.gethostbyaddr(ip_addr)[0]
+ log.debug("derived host_name for host \"{}\": {}".format(
+ host, host_name))
+ except (socket.gaierror, socket.herror) as e:
+ log.warn("Could not derive host_name for {}"
+ ", $host_name will be empty. Error is: {}".format(host, e))
+ # in case of error provide empty value
+ host_name = ''
+
+ try:
+ host_fqdn = socket.getfqdn(host)
+ log.debug("derived host_fqdn for host \"{}\": {}".format(
+ host, host_fqdn))
+ except socket.herror as e:
+ log.warn("Could not derive host_fqdn for {}"
+ ", $host_fqdn will be empty. Error is: {}".format(host, e))
+ # in case of error provide empty value
+ host_fqdn = ''
+
+ return (host, host_name, host_fqdn)
+
+
+# check whether addr is IPv6
+try:
+ # python 3.3+
+ import ipaddress
+
+ def is_ipv6(addr):
+ try:
+ return ipaddress.ip_address(addr).version == 6
+ except ValueError:
+ return False
+except ImportError:
+ # fallback for older python versions
+ def is_ipv6(addr):
+ try:
+ socket.inet_aton(addr)
+ return False
+ except socket.error:
+ pass
+ try:
+ socket.inet_pton(socket.AF_INET6, addr)
+ return True
+ except socket.error:
+ pass
+ return False
diff --git a/cdist/util/remoteutil.py b/cdist/util/remoteutil.py
new file mode 100644
index 00000000..c18d6705
--- /dev/null
+++ b/cdist/util/remoteutil.py
@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+#
+# 2016 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see .
+#
+#
+
+
+def inspect_ssh_mux_opts():
+ """Inspect whether or not ssh supports multiplexing options.
+
+ Return string containing multiplexing options if supported.
+ If ControlPath is supported then placeholder for that path is
+ specified and can be used for final string formatting.
+ For example, this function can return string:
+ "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
+ Then it can be formatted:
+ mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
+ """
+ import subprocess
+
+ wanted_mux_opts = {
+ "ControlPath": "{}",
+ "ControlMaster": "auto",
+ "ControlPersist": "125",
+ }
+ mux_opts = " ".join([" -o {}={}".format(
+ x, wanted_mux_opts[x]) for x in wanted_mux_opts])
+ try:
+ subprocess.check_output("ssh {}".format(mux_opts),
+ stderr=subprocess.STDOUT, shell=True)
+ except subprocess.CalledProcessError as e:
+ subproc_output = e.output.decode().lower()
+ if "bad configuration option" in subproc_output:
+ return ""
+ return mux_opts
diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh
index 97b22473..4b94b858 100755
--- a/docs/src/cdist-reference.rst.sh
+++ b/docs/src/cdist-reference.rst.sh
@@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp).
+
+CDIST_BETA
+ Enable beta functionalities.
eof
diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst
index 5daedcd4..08c856b1 100644
--- a/docs/src/man1/cdist.rst
+++ b/docs/src/man1/cdist.rst
@@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp).
+CDIST_BETA
+ Enable beta functionalities.
+
EXIT STATUS
-----------
The following exit values shall be returned:
diff --git a/scripts/cdist b/scripts/cdist
index 4ff64bb5..498091b8 100755
--- a/scripts/cdist
+++ b/scripts/cdist
@@ -1,7 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
-# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
+# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
@@ -24,182 +24,25 @@
import collections
import logging
-# list of beta sub-commands
-BETA_COMMANDS = ['install', ]
-# list of beta arguments for sub-commands
-BETA_ARGS = {
- 'config': ['jobs', ],
-}
-
-
-def check_positive_int(value):
- import argparse
-
- try:
- val = int(value)
- except ValueError as e:
- raise argparse.ArgumentTypeError(
- "{} is invalid int value".format(value))
- if val <= 0:
- raise argparse.ArgumentTypeError(
- "{} is invalid positive int value".format(val))
- return val
-
-
-def check_beta(args_dict):
- if 'beta' not in args_dict:
- args_dict['beta'] = False
- # Check only if beta is not enabled: if beta option is specified then
- # raise error.
- if not args_dict['beta']:
- cmd = args_dict['command']
- # first check if command is beta
- if cmd in BETA_COMMANDS:
- raise cdist.CdistBetaRequired(cmd)
- # then check if command's argument is beta
- if cmd in BETA_ARGS:
- for arg in BETA_ARGS[cmd]:
- if arg in args_dict and args_dict[arg]:
- raise cdist.CdistBetaRequired(cmd, arg)
-
-
-_verbosity_level = {
- 0: logging.ERROR,
- 1: logging.WARNING,
- 2: logging.INFO,
-}
-_verbosity_level = collections.defaultdict(
- lambda: logging.DEBUG, _verbosity_level)
-
def commandline():
"""Parse command line"""
- import argparse
+ import cdist.argparse
import cdist.banner
import cdist.config
import cdist.install
import cdist.shell
import shutil
import os
- import multiprocessing
-
- # Construct parser others can reuse
- parser = {}
- # Options _all_ parsers have in common
- parser['loglevel'] = argparse.ArgumentParser(add_help=False)
- parser['loglevel'].add_argument(
- '-d', '--debug',
- help=('Set log level to debug (deprecated, use -vvv instead)'),
- action='store_true', default=False)
- parser['loglevel'].add_argument(
- '-v', '--verbose',
- help=('Increase log level, be more verbose. Use it more than once '
- 'to increase log level. The order of levels from the lowest '
- 'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
- action='count', default=0)
-
- # Main subcommand parser
- parser['main'] = argparse.ArgumentParser(
- description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
- parser['main'].add_argument(
- '-V', '--version', help='Show version', action='version',
- version='%(prog)s ' + cdist.VERSION)
- parser['sub'] = parser['main'].add_subparsers(
- title="Commands", dest="command")
-
- # Banner
- parser['banner'] = parser['sub'].add_parser(
- 'banner', parents=[parser['loglevel']])
- parser['banner'].set_defaults(func=cdist.banner.banner)
-
- # Config
- parser['config'] = parser['sub'].add_parser(
- 'config', parents=[parser['loglevel']])
- parser['config'].add_argument(
- 'host', nargs='*', help='host(s) to operate on')
- parser['config'].add_argument(
- '-b', '--enable-beta',
- help=('Enable beta functionalities. Beta functionalities '
- 'include the following options: -j/--jobs.'),
- action='store_true', dest='beta', default=False)
- parser['config'].add_argument(
- '-c', '--conf-dir',
- help=('Add configuration directory (can be repeated, '
- 'last one wins)'), action='append')
- parser['config'].add_argument(
- '-f', '--file',
- help=('Read additional hosts to operate on from specified file '
- 'or from stdin if \'-\' (each host on separate line). '
- 'If no host or host file is specified then, by default, '
- 'read hosts from stdin.'),
- dest='hostfile', required=False)
- parser['config'].add_argument(
- '-i', '--initial-manifest',
- help='Path to a cdist manifest or \'-\' to read from stdin.',
- dest='manifest', required=False)
- parser['config'].add_argument(
- '-j', '--jobs', nargs='?', type=check_positive_int,
- help=('Specify the maximum number of parallel jobs, currently '
- 'only global explorers are supported (currently in beta'),
- action='store', dest='jobs',
- const=multiprocessing.cpu_count())
- parser['config'].add_argument(
- '-n', '--dry-run',
- help='Do not execute code', action='store_true')
- parser['config'].add_argument(
- '-o', '--out-dir',
- help='Directory to save cdist output in', dest="out_path")
- parser['config'].add_argument(
- '-p', '--parallel',
- help='Operate on multiple hosts in parallel',
- action='store_true', dest='parallel')
- parser['config'].add_argument(
- '-s', '--sequential',
- help='Operate on multiple hosts sequentially (default)',
- action='store_false', dest='parallel')
- # remote-copy and remote-exec defaults are environment variables
- # if set; if not then None - these will be futher handled after
- # parsing to determine implementation default
- parser['config'].add_argument(
- '--remote-copy',
- help='Command to use for remote copy (should behave like scp)',
- action='store', dest='remote_copy',
- default=os.environ.get('CDIST_REMOTE_COPY'))
- parser['config'].add_argument(
- '--remote-exec',
- help=('Command to use for remote execution '
- '(should behave like ssh)'),
- action='store', dest='remote_exec',
- default=os.environ.get('CDIST_REMOTE_EXEC'))
- parser['config'].set_defaults(func=cdist.config.Config.commandline)
-
- # Install
- parser['install'] = parser['sub'].add_parser('install', add_help=False,
- parents=[parser['config']])
- parser['install'].set_defaults(func=cdist.install.Install.commandline)
-
- # Shell
- parser['shell'] = parser['sub'].add_parser(
- 'shell', parents=[parser['loglevel']])
- parser['shell'].add_argument(
- '-s', '--shell',
- help=('Select shell to use, defaults to current shell. Used shell'
- ' should be POSIX compatible shell.'))
- parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
-
- for p in parser:
- parser[p].epilog = (
- "Get cdist at http://www.nico.schottelius.org/software/cdist/")
+ parser = cdist.argparse.get_parsers()
args = parser['main'].parse_args(sys.argv[1:])
# Loglevels are handled globally in here
- if args.debug:
- log.warning("-d/--debug is deprecated, use -vvv instead")
- args.verbose = 3
-
- logging.root.setLevel(_verbosity_level[args.verbose])
+ retval = cdist.argparse.handle_loglevel(args)
+ if retval:
+ log.warning(retval)
log.debug(args)
log.info("version %s" % cdist.VERSION)
@@ -219,17 +62,16 @@ def commandline():
parser['main'].print_help()
sys.exit(0)
- check_beta(vars(args))
+ cdist.argparse.check_beta(vars(args))
args.func(args)
if __name__ == "__main__":
- # Sys is needed for sys.exit()
import sys
cdistpythonversion = '3.2'
if sys.version < cdistpythonversion:
- print('Python >= ' + cdistpythonversion +
- ' is required on the source host.', file=sys.stderr)
+ print('Python >= {} is required on the source host.'.format(
+ cdistpythonversion), file=sys.stderr)
sys.exit(1)
exit_code = 0