Merge pull request #504 from darko-poljak/the-good-the-bad-and-the-ugly
The Good, the Bad and the Ugly
This commit is contained in:
Darko Poljak
GitHub
commit
bc5f6c8923
|
|
@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error):
|
|||
err_msg = ("\'{}\' command is beta, but beta is "
|
||||
"not enabled. If you want to use it please enable beta "
|
||||
"functionalities by using the -b/--enable-beta command "
|
||||
"line flag.")
|
||||
"line flag or setting CDIST_BETA env var.")
|
||||
fmt_args = [self.command, ]
|
||||
else:
|
||||
err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta "
|
||||
"is not enabled. If you want to use it please enable "
|
||||
"beta functionalities by using the -b/--enable-beta "
|
||||
"command line flag.")
|
||||
"command line flag or setting CDIST_BETA env var.")
|
||||
fmt_args = [self.arg, self.command, ]
|
||||
return err_msg.format(*fmt_args)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,208 @@
|
|||
import argparse
|
||||
import cdist
|
||||
import multiprocessing
|
||||
import os
|
||||
import logging
|
||||
import collections
|
||||
|
||||
|
||||
# set of beta sub-commands
|
||||
BETA_COMMANDS = set(('install', ))
|
||||
# set of beta arguments for sub-commands
|
||||
BETA_ARGS = {
|
||||
'config': set(('jobs', )),
|
||||
}
|
||||
EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
|
||||
# Parser others can reuse
|
||||
parser = None
|
||||
|
||||
|
||||
_verbosity_level = {
|
||||
0: logging.ERROR,
|
||||
1: logging.WARNING,
|
||||
2: logging.INFO,
|
||||
}
|
||||
_verbosity_level = collections.defaultdict(
|
||||
lambda: logging.DEBUG, _verbosity_level)
|
||||
|
||||
|
||||
def add_beta_command(cmd):
|
||||
BETA_COMMANDS.add(cmd)
|
||||
|
||||
|
||||
def add_beta_arg(cmd, arg):
|
||||
if cmd in BETA_ARGS:
|
||||
if arg not in BETA_ARGS[cmd]:
|
||||
BETA_ARGS[cmd].append(arg)
|
||||
else:
|
||||
BETA_ARGS[cmd] = set((arg, ))
|
||||
|
||||
|
||||
def check_beta(args_dict):
|
||||
if 'beta' not in args_dict:
|
||||
args_dict['beta'] = False
|
||||
# Check only if beta is not enabled: if beta option is specified then
|
||||
# raise error.
|
||||
if not args_dict['beta']:
|
||||
cmd = args_dict['command']
|
||||
# first check if command is beta
|
||||
if cmd in BETA_COMMANDS:
|
||||
raise cdist.CdistBetaRequired(cmd)
|
||||
# then check if some command's argument is beta
|
||||
if cmd in BETA_ARGS:
|
||||
for arg in BETA_ARGS[cmd]:
|
||||
if arg in args_dict and args_dict[arg]:
|
||||
raise cdist.CdistBetaRequired(cmd, arg)
|
||||
|
||||
|
||||
def check_positive_int(value):
|
||||
import argparse
|
||||
|
||||
try:
|
||||
val = int(value)
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{} is invalid int value".format(value))
|
||||
if val <= 0:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{} is invalid positive int value".format(val))
|
||||
return val
|
||||
|
||||
|
||||
def get_parsers():
|
||||
global parser
|
||||
|
||||
# Construct parser others can reuse
|
||||
if parser:
|
||||
return parser
|
||||
else:
|
||||
parser = {}
|
||||
# Options _all_ parsers have in common
|
||||
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['loglevel'].add_argument(
|
||||
'-d', '--debug',
|
||||
help=('Set log level to debug (deprecated, use -vvv instead)'),
|
||||
action='store_true', default=False)
|
||||
parser['loglevel'].add_argument(
|
||||
'-v', '--verbose',
|
||||
help=('Increase log level, be more verbose. Use it more than once '
|
||||
'to increase log level. The order of levels from the lowest '
|
||||
'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
|
||||
action='count', default=0)
|
||||
|
||||
parser['beta'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['beta'].add_argument(
|
||||
'-b', '--beta',
|
||||
help=('Enable beta functionalities. '
|
||||
'Can also be enabled using CDIST_BETA env var.'),
|
||||
action='store_true', dest='beta',
|
||||
default='CDIST_BETA' in os.environ)
|
||||
|
||||
# Main subcommand parser
|
||||
parser['main'] = argparse.ArgumentParser(
|
||||
description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
|
||||
parser['main'].add_argument(
|
||||
'-V', '--version', help='Show version', action='version',
|
||||
version='%(prog)s ' + cdist.VERSION)
|
||||
parser['sub'] = parser['main'].add_subparsers(
|
||||
title="Commands", dest="command")
|
||||
|
||||
# Banner
|
||||
parser['banner'] = parser['sub'].add_parser(
|
||||
'banner', parents=[parser['loglevel']])
|
||||
parser['banner'].set_defaults(func=cdist.banner.banner)
|
||||
|
||||
# Config
|
||||
parser['config_main'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['config_main'].add_argument(
|
||||
'-c', '--conf-dir',
|
||||
help=('Add configuration directory (can be repeated, '
|
||||
'last one wins)'), action='append')
|
||||
parser['config_main'].add_argument(
|
||||
'-i', '--initial-manifest',
|
||||
help='path to a cdist manifest or \'-\' to read from stdin.',
|
||||
dest='manifest', required=False)
|
||||
parser['config_main'].add_argument(
|
||||
'-j', '--jobs', nargs='?',
|
||||
type=check_positive_int,
|
||||
help=('Specify the maximum number of parallel jobs, currently '
|
||||
'only global explorers are supported'),
|
||||
action='store', dest='jobs',
|
||||
const=multiprocessing.cpu_count())
|
||||
parser['config_main'].add_argument(
|
||||
'-n', '--dry-run',
|
||||
help='do not execute code', action='store_true')
|
||||
parser['config_main'].add_argument(
|
||||
'-o', '--out-dir',
|
||||
help='directory to save cdist output in', dest="out_path")
|
||||
|
||||
# remote-copy and remote-exec defaults are environment variables
|
||||
# if set; if not then None - these will be futher handled after
|
||||
# parsing to determine implementation default
|
||||
parser['config_main'].add_argument(
|
||||
'--remote-copy',
|
||||
help='Command to use for remote copy (should behave like scp)',
|
||||
action='store', dest='remote_copy',
|
||||
default=os.environ.get('CDIST_REMOTE_COPY'))
|
||||
parser['config_main'].add_argument(
|
||||
'--remote-exec',
|
||||
help=('Command to use for remote execution '
|
||||
'(should behave like ssh)'),
|
||||
action='store', dest='remote_exec',
|
||||
default=os.environ.get('CDIST_REMOTE_EXEC'))
|
||||
|
||||
# Config
|
||||
parser['config_args'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['config_args'].add_argument(
|
||||
'host', nargs='*', help='host(s) to operate on')
|
||||
parser['config_args'].add_argument(
|
||||
'-f', '--file',
|
||||
help=('Read additional hosts to operate on from specified file '
|
||||
'or from stdin if \'-\' (each host on separate line). '
|
||||
'If no host or host file is specified then, by default, '
|
||||
'read hosts from stdin.'),
|
||||
dest='hostfile', required=False)
|
||||
parser['config_args'].add_argument(
|
||||
'-p', '--parallel',
|
||||
help='operate on multiple hosts in parallel',
|
||||
action='store_true', dest='parallel')
|
||||
parser['config_args'].add_argument(
|
||||
'-s', '--sequential',
|
||||
help='operate on multiple hosts sequentially (default)',
|
||||
action='store_false', dest='parallel')
|
||||
parser['config'] = parser['sub'].add_parser(
|
||||
'config', parents=[parser['loglevel'], parser['beta'],
|
||||
parser['config_main'],
|
||||
parser['config_args']])
|
||||
parser['config'].set_defaults(func=cdist.config.Config.commandline)
|
||||
|
||||
# Install
|
||||
parser['install'] = parser['sub'].add_parser('install', add_help=False,
|
||||
parents=[parser['config']])
|
||||
parser['install'].set_defaults(func=cdist.install.Install.commandline)
|
||||
|
||||
# Shell
|
||||
parser['shell'] = parser['sub'].add_parser(
|
||||
'shell', parents=[parser['loglevel']])
|
||||
parser['shell'].add_argument(
|
||||
'-s', '--shell',
|
||||
help=('Select shell to use, defaults to current shell. Used shell'
|
||||
' should be POSIX compatible shell.'))
|
||||
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
|
||||
|
||||
for p in parser:
|
||||
parser[p].epilog = EPILOG
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def handle_loglevel(args):
|
||||
if args.debug:
|
||||
retval = "-d/--debug is deprecated, use -vvv instead"
|
||||
args.verbose = 3
|
||||
else:
|
||||
retval = None
|
||||
|
||||
logging.root.setLevel(_verbosity_level[args.verbose])
|
||||
|
||||
return retval
|
||||
177
cdist/config.py
177
cdist/config.py
|
|
@ -22,50 +22,21 @@
|
|||
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import time
|
||||
import pprint
|
||||
import itertools
|
||||
import tempfile
|
||||
import socket
|
||||
|
||||
import cdist
|
||||
import cdist.hostsource
|
||||
|
||||
import cdist.exec.local
|
||||
import cdist.exec.remote
|
||||
import cdist.util.ipaddr as ipaddr
|
||||
|
||||
from cdist import core
|
||||
|
||||
|
||||
def inspect_ssh_mux_opts():
|
||||
"""Inspect whether or not ssh supports multiplexing options.
|
||||
|
||||
Return string containing multiplexing options if supported.
|
||||
If ControlPath is supported then placeholder for that path is
|
||||
specified and can be used for final string formatting.
|
||||
For example, this function can return string:
|
||||
"-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
|
||||
Then it can be formatted:
|
||||
mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
wanted_mux_opts = {
|
||||
"ControlPath": "{}",
|
||||
"ControlMaster": "auto",
|
||||
"ControlPersist": "125",
|
||||
}
|
||||
mux_opts = " ".join([" -o {}={}".format(
|
||||
x, wanted_mux_opts[x]) for x in wanted_mux_opts])
|
||||
try:
|
||||
subprocess.check_output("ssh {}".format(mux_opts),
|
||||
stderr=subprocess.STDOUT, shell=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
subproc_output = e.output.decode().lower()
|
||||
if "bad configuration option" in subproc_output:
|
||||
return ""
|
||||
return mux_opts
|
||||
from cdist.util.remoteutil import inspect_ssh_mux_opts
|
||||
|
||||
|
||||
class Config(object):
|
||||
|
|
@ -89,55 +60,17 @@ class Config(object):
|
|||
self.local.create_files_dirs()
|
||||
self.remote.create_files_dirs()
|
||||
|
||||
@staticmethod
|
||||
def hostfile_process_line(line):
|
||||
"""Return host from read line or None if no host present."""
|
||||
if not line:
|
||||
return None
|
||||
# remove comment if present
|
||||
comment_index = line.find('#')
|
||||
if comment_index >= 0:
|
||||
host = line[:comment_index]
|
||||
else:
|
||||
host = line
|
||||
# remove leading and trailing whitespaces
|
||||
host = host.strip()
|
||||
# skip empty lines
|
||||
if host:
|
||||
return host
|
||||
else:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def hosts(source):
|
||||
"""Yield hosts from source.
|
||||
Source can be a sequence or filename (stdin if \'-\').
|
||||
In case of filename each line represents one host.
|
||||
"""
|
||||
if isinstance(source, str):
|
||||
import fileinput
|
||||
try:
|
||||
for host in fileinput.input(files=(source)):
|
||||
host = Config.hostfile_process_line(host)
|
||||
if host:
|
||||
yield host
|
||||
except (IOError, OSError, UnicodeError) as e:
|
||||
raise cdist.Error(
|
||||
"Error reading hosts from file \'{}\': {}".format(
|
||||
source, e))
|
||||
else:
|
||||
if source:
|
||||
for host in source:
|
||||
yield host
|
||||
try:
|
||||
yield from cdist.hostsource.HostSource(source)()
|
||||
except (IOError, OSError, UnicodeError) as e:
|
||||
raise cdist.Error(
|
||||
"Error reading hosts from \'{}\': {}".format(
|
||||
source, e))
|
||||
|
||||
@classmethod
|
||||
def commandline(cls, args):
|
||||
"""Configure remote system"""
|
||||
import multiprocessing
|
||||
|
||||
# FIXME: Refactor relict - remove later
|
||||
log = logging.getLogger("cdist")
|
||||
|
||||
def _check_and_prepare_args(cls, args):
|
||||
if args.manifest == '-' and args.hostfile == '-':
|
||||
raise cdist.Error(("Cannot read both, manifest and host file, "
|
||||
"from stdin"))
|
||||
|
|
@ -162,10 +95,6 @@ class Config(object):
|
|||
import atexit
|
||||
atexit.register(lambda: os.remove(initial_manifest_temp_path))
|
||||
|
||||
process = {}
|
||||
failed_hosts = []
|
||||
time_start = time.time()
|
||||
|
||||
# default remote cmd patterns
|
||||
args.remote_exec_pattern = None
|
||||
args.remote_copy_pattern = None
|
||||
|
|
@ -182,10 +111,29 @@ class Config(object):
|
|||
if args_dict['remote_copy'] is None:
|
||||
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
|
||||
|
||||
@classmethod
|
||||
def _base_root_path(cls, args):
|
||||
if args.out_path:
|
||||
base_root_path = args.out_path
|
||||
else:
|
||||
base_root_path = tempfile.mkdtemp()
|
||||
return base_root_path
|
||||
|
||||
@classmethod
|
||||
def commandline(cls, args):
|
||||
"""Configure remote system"""
|
||||
import multiprocessing
|
||||
|
||||
# FIXME: Refactor relict - remove later
|
||||
log = logging.getLogger("cdist")
|
||||
|
||||
cls._check_and_prepare_args(args)
|
||||
|
||||
process = {}
|
||||
failed_hosts = []
|
||||
time_start = time.time()
|
||||
|
||||
base_root_path = cls._base_root_path(args)
|
||||
|
||||
hostcnt = 0
|
||||
for host in itertools.chain(cls.hosts(args.host),
|
||||
|
|
@ -227,6 +175,24 @@ class Config(object):
|
|||
raise cdist.Error("Failed to configure the following hosts: " +
|
||||
" ".join(failed_hosts))
|
||||
|
||||
@classmethod
|
||||
def _resolve_remote_cmds(cls, args, host_base_path):
|
||||
control_path = os.path.join(host_base_path, "ssh-control-path")
|
||||
# If we constructed patterns for remote commands then there is
|
||||
# placeholder for ssh ControlPath, format it and we have unique
|
||||
# ControlPath for each host.
|
||||
#
|
||||
# If not then use args.remote_exec/copy that user specified.
|
||||
if args.remote_exec_pattern:
|
||||
remote_exec = args.remote_exec_pattern.format(control_path)
|
||||
else:
|
||||
remote_exec = args.remote_exec
|
||||
if args.remote_copy_pattern:
|
||||
remote_copy = args.remote_copy_pattern.format(control_path)
|
||||
else:
|
||||
remote_copy = args.remote_copy
|
||||
return (remote_exec, remote_copy, )
|
||||
|
||||
@classmethod
|
||||
def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
|
||||
"""Configure ONE system"""
|
||||
|
|
@ -234,57 +200,14 @@ class Config(object):
|
|||
log = logging.getLogger(host)
|
||||
|
||||
try:
|
||||
control_path = os.path.join(host_base_path, "ssh-control-path")
|
||||
# If we constructed patterns for remote commands then there is
|
||||
# placeholder for ssh ControlPath, format it and we have unique
|
||||
# ControlPath for each host.
|
||||
#
|
||||
# If not then use args.remote_exec/copy that user specified.
|
||||
if args.remote_exec_pattern:
|
||||
remote_exec = args.remote_exec_pattern.format(control_path)
|
||||
else:
|
||||
remote_exec = args.remote_exec
|
||||
if args.remote_copy_pattern:
|
||||
remote_copy = args.remote_copy_pattern.format(control_path)
|
||||
else:
|
||||
remote_copy = args.remote_copy
|
||||
remote_exec, remote_copy = cls._resolve_remote_cmds(
|
||||
args, host_base_path)
|
||||
log.debug("remote_exec for host \"{}\": {}".format(
|
||||
host, remote_exec))
|
||||
log.debug("remote_copy for host \"{}\": {}".format(
|
||||
host, remote_copy))
|
||||
|
||||
try:
|
||||
# getaddrinfo returns a list of 5-tuples:
|
||||
# (family, type, proto, canonname, sockaddr)
|
||||
# where sockaddr is:
|
||||
# (address, port) for AF_INET,
|
||||
# (address, port, flow_info, scopeid) for AF_INET6
|
||||
ip_addr = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM)[0][4][0]
|
||||
# gethostbyaddr returns triple
|
||||
# (hostname, aliaslist, ipaddrlist)
|
||||
host_name = socket.gethostbyaddr(ip_addr)[0]
|
||||
log.debug("derived host_name for host \"{}\": {}".format(
|
||||
host, host_name))
|
||||
except (socket.gaierror, socket.herror) as e:
|
||||
log.warn("Could not derive host_name for {}"
|
||||
", $host_name will be empty. Error is: {}".format(
|
||||
host, e))
|
||||
# in case of error provide empty value
|
||||
host_name = ''
|
||||
|
||||
try:
|
||||
host_fqdn = socket.getfqdn(host)
|
||||
log.debug("derived host_fqdn for host \"{}\": {}".format(
|
||||
host, host_fqdn))
|
||||
except socket.herror as e:
|
||||
log.warn("Could not derive host_fqdn for {}"
|
||||
", $host_fqdn will be empty. Error is: {}".format(
|
||||
host, e))
|
||||
# in case of error provide empty value
|
||||
host_fqdn = ''
|
||||
|
||||
target_host = (host, host_name, host_fqdn)
|
||||
target_host = ipaddr.resolve_target_addresses(host)
|
||||
log.debug("target_host: {}".format(target_host))
|
||||
|
||||
local = cdist.exec.local.Local(
|
||||
|
|
|
|||
|
|
@ -149,14 +149,9 @@ class Explorer(object):
|
|||
def transfer_global_explorers(self):
|
||||
"""Transfer the global explorers to the remote side."""
|
||||
self.remote.mkdir(self.remote.global_explorer_path)
|
||||
if self.jobs is None:
|
||||
self.remote.transfer(self.local.global_explorer_path,
|
||||
self.remote.global_explorer_path)
|
||||
else:
|
||||
self.remote.transfer_dir_parallel(
|
||||
self.local.global_explorer_path,
|
||||
self.remote.global_explorer_path,
|
||||
self.jobs)
|
||||
self.remote.transfer(self.local.global_explorer_path,
|
||||
self.remote.global_explorer_path,
|
||||
self.jobs)
|
||||
self.remote.run(["chmod", "0700",
|
||||
"%s/*" % (self.remote.global_explorer_path)])
|
||||
|
||||
|
|
|
|||
|
|
@ -30,40 +30,13 @@ import multiprocessing
|
|||
|
||||
import cdist
|
||||
import cdist.exec.util as exec_util
|
||||
|
||||
|
||||
# check whether addr is IPv6
|
||||
try:
|
||||
# python 3.3+
|
||||
import ipaddress
|
||||
|
||||
def _is_ipv6(addr):
|
||||
try:
|
||||
return ipaddress.ip_address(addr).version == 6
|
||||
except ValueError:
|
||||
return False
|
||||
except ImportError:
|
||||
# fallback for older python versions
|
||||
import socket
|
||||
|
||||
def _is_ipv6(addr):
|
||||
try:
|
||||
socket.inet_aton(addr)
|
||||
return False
|
||||
except socket.error:
|
||||
pass
|
||||
try:
|
||||
socket.inet_pton(socket.AF_INET6, addr)
|
||||
return True
|
||||
except socket.error:
|
||||
pass
|
||||
return False
|
||||
import cdist.util.ipaddr as ipaddr
|
||||
|
||||
|
||||
def _wrap_addr(addr):
|
||||
"""If addr is IPv6 then return addr wrapped between '[' and ']',
|
||||
otherwise return it intact."""
|
||||
if _is_ipv6(addr):
|
||||
if ipaddr.is_ipv6(addr):
|
||||
return "".join(("[", addr, "]", ))
|
||||
else:
|
||||
return addr
|
||||
|
|
@ -145,57 +118,59 @@ class Remote(object):
|
|||
self.log.debug("Remote mkdir: %s", path)
|
||||
self.run(["mkdir", "-p", path])
|
||||
|
||||
def transfer(self, source, destination):
|
||||
def transfer(self, source, destination, jobs=None):
|
||||
"""Transfer a file or directory to the remote side."""
|
||||
self.log.debug("Remote transfer: %s -> %s", source, destination)
|
||||
self.rmdir(destination)
|
||||
if os.path.isdir(source):
|
||||
self.mkdir(destination)
|
||||
for f in glob.glob1(source, '*'):
|
||||
command = self._copy.split()
|
||||
path = os.path.join(source, f)
|
||||
command.extend([path, '{0}:{1}'.format(
|
||||
_wrap_addr(self.target_host[0]), destination)])
|
||||
self._run_command(command)
|
||||
if jobs:
|
||||
self._transfer_dir_parallel(source, destination, jobs)
|
||||
else:
|
||||
self._transfer_dir_sequential(source, destination)
|
||||
elif jobs:
|
||||
raise cdist.Error("Source {} is not a directory".format(source))
|
||||
else:
|
||||
command = self._copy.split()
|
||||
command.extend([source, '{0}:{1}'.format(
|
||||
_wrap_addr(self.target_host[0]), destination)])
|
||||
self._run_command(command)
|
||||
|
||||
def transfer_dir_parallel(self, source, destination, jobs):
|
||||
"""Transfer a directory to the remote side in parallel mode."""
|
||||
self.log.debug("Remote transfer: %s -> %s", source, destination)
|
||||
self.rmdir(destination)
|
||||
if os.path.isdir(source):
|
||||
self.mkdir(destination)
|
||||
self.log.info("Remote transfer in {} parallel jobs".format(
|
||||
jobs))
|
||||
self.log.debug("Multiprocessing start method is {}".format(
|
||||
multiprocessing.get_start_method()))
|
||||
self.log.debug(("Starting multiprocessing Pool for parallel "
|
||||
"remote transfer"))
|
||||
with multiprocessing.Pool(jobs) as pool:
|
||||
self.log.debug("Starting async for parallel transfer")
|
||||
commands = []
|
||||
for f in glob.glob1(source, '*'):
|
||||
command = self._copy.split()
|
||||
path = os.path.join(source, f)
|
||||
command.extend([path, '{0}:{1}'.format(
|
||||
_wrap_addr(self.target_host[0]), destination)])
|
||||
commands.append(command)
|
||||
results = [
|
||||
pool.apply_async(self._run_command, (cmd,))
|
||||
for cmd in commands
|
||||
]
|
||||
def _transfer_dir_sequential(self, source, destination):
|
||||
for f in glob.glob1(source, '*'):
|
||||
command = self._copy.split()
|
||||
path = os.path.join(source, f)
|
||||
command.extend([path, '{0}:{1}'.format(
|
||||
_wrap_addr(self.target_host[0]), destination)])
|
||||
self._run_command(command)
|
||||
|
||||
self.log.debug("Waiting async results for parallel transfer")
|
||||
for r in results:
|
||||
r.get() # self._run_command returns None
|
||||
self.log.debug(("Multiprocessing for parallel transfer "
|
||||
"finished"))
|
||||
else:
|
||||
raise cdist.Error("Source {} is not a directory".format(source))
|
||||
def _transfer_dir_parallel(self, source, destination, jobs):
|
||||
"""Transfer a directory to the remote side in parallel mode."""
|
||||
self.log.info("Remote transfer in {} parallel jobs".format(
|
||||
jobs))
|
||||
self.log.debug("Multiprocessing start method is {}".format(
|
||||
multiprocessing.get_start_method()))
|
||||
self.log.debug(("Starting multiprocessing Pool for parallel "
|
||||
"remote transfer"))
|
||||
with multiprocessing.Pool(jobs) as pool:
|
||||
self.log.debug("Starting async for parallel transfer")
|
||||
commands = []
|
||||
for f in glob.glob1(source, '*'):
|
||||
command = self._copy.split()
|
||||
path = os.path.join(source, f)
|
||||
command.extend([path, '{0}:{1}'.format(
|
||||
_wrap_addr(self.target_host[0]), destination)])
|
||||
commands.append(command)
|
||||
results = [
|
||||
pool.apply_async(self._run_command, (cmd,))
|
||||
for cmd in commands
|
||||
]
|
||||
|
||||
self.log.debug("Waiting async results for parallel transfer")
|
||||
for r in results:
|
||||
r.get() # self._run_command returns None
|
||||
self.log.debug(("Multiprocessing for parallel transfer "
|
||||
"finished"))
|
||||
|
||||
def run_script(self, script, env=None, return_output=False):
|
||||
"""Run the given script with the given environment on the remote side.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
# cdist is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# cdist is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
#
|
||||
|
||||
import fileinput
|
||||
|
||||
|
||||
class HostSource(object):
|
||||
"""
|
||||
Host source object.
|
||||
Source can be a sequence or filename (stdin if \'-\').
|
||||
In case of filename each line represents one host.
|
||||
"""
|
||||
def __init__(self, source):
|
||||
self.source = source
|
||||
|
||||
def _process_file_line(self, line):
|
||||
"""Return host from read line or None if no host present."""
|
||||
if not line:
|
||||
return None
|
||||
# remove comment if present
|
||||
comment_index = line.find('#')
|
||||
if comment_index >= 0:
|
||||
host = line[:comment_index]
|
||||
else:
|
||||
host = line
|
||||
# remove leading and trailing whitespaces
|
||||
host = host.strip()
|
||||
# skip empty lines
|
||||
if host:
|
||||
return host
|
||||
else:
|
||||
return None
|
||||
|
||||
def _hosts_from_sequence(self):
|
||||
for host in self.source:
|
||||
yield host
|
||||
|
||||
def _hosts_from_file(self):
|
||||
for line in fileinput.input(files=(self.source)):
|
||||
host = self._process_file_line(line)
|
||||
if host:
|
||||
yield host
|
||||
|
||||
def hosts(self):
|
||||
if not self.source:
|
||||
return
|
||||
|
||||
if isinstance(self.source, str):
|
||||
yield from self._hosts_from_file()
|
||||
else:
|
||||
yield from self._hosts_from_sequence()
|
||||
|
||||
def __call__(self):
|
||||
yield from self.hosts()
|
||||
|
|
@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase):
|
|||
source_file_name = os.path.split(source_file)[-1]
|
||||
filenames.append(source_file_name)
|
||||
target = self.mkdtemp(dir=self.temp_dir)
|
||||
self.remote.transfer_dir_parallel(source, target,
|
||||
multiprocessing.cpu_count())
|
||||
self.remote.transfer(source, target,
|
||||
multiprocessing.cpu_count())
|
||||
# test if the payload files are in the target directory
|
||||
for filename in filenames:
|
||||
self.assertTrue(os.path.isfile(os.path.join(target, filename)))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,83 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
# cdist is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# cdist is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
#
|
||||
|
||||
import socket
|
||||
import logging
|
||||
|
||||
|
||||
def resolve_target_addresses(host):
|
||||
log = logging.getLogger(host)
|
||||
try:
|
||||
# getaddrinfo returns a list of 5-tuples:
|
||||
# (family, type, proto, canonname, sockaddr)
|
||||
# where sockaddr is:
|
||||
# (address, port) for AF_INET,
|
||||
# (address, port, flow_info, scopeid) for AF_INET6
|
||||
ip_addr = socket.getaddrinfo(
|
||||
host, None, type=socket.SOCK_STREAM)[0][4][0]
|
||||
# gethostbyaddr returns triple
|
||||
# (hostname, aliaslist, ipaddrlist)
|
||||
host_name = socket.gethostbyaddr(ip_addr)[0]
|
||||
log.debug("derived host_name for host \"{}\": {}".format(
|
||||
host, host_name))
|
||||
except (socket.gaierror, socket.herror) as e:
|
||||
log.warn("Could not derive host_name for {}"
|
||||
", $host_name will be empty. Error is: {}".format(host, e))
|
||||
# in case of error provide empty value
|
||||
host_name = ''
|
||||
|
||||
try:
|
||||
host_fqdn = socket.getfqdn(host)
|
||||
log.debug("derived host_fqdn for host \"{}\": {}".format(
|
||||
host, host_fqdn))
|
||||
except socket.herror as e:
|
||||
log.warn("Could not derive host_fqdn for {}"
|
||||
", $host_fqdn will be empty. Error is: {}".format(host, e))
|
||||
# in case of error provide empty value
|
||||
host_fqdn = ''
|
||||
|
||||
return (host, host_name, host_fqdn)
|
||||
|
||||
|
||||
# check whether addr is IPv6
|
||||
try:
|
||||
# python 3.3+
|
||||
import ipaddress
|
||||
|
||||
def is_ipv6(addr):
|
||||
try:
|
||||
return ipaddress.ip_address(addr).version == 6
|
||||
except ValueError:
|
||||
return False
|
||||
except ImportError:
|
||||
# fallback for older python versions
|
||||
def is_ipv6(addr):
|
||||
try:
|
||||
socket.inet_aton(addr)
|
||||
return False
|
||||
except socket.error:
|
||||
pass
|
||||
try:
|
||||
socket.inet_pton(socket.AF_INET6, addr)
|
||||
return True
|
||||
except socket.error:
|
||||
pass
|
||||
return False
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
# cdist is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# cdist is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
||||
#
|
||||
#
|
||||
|
||||
|
||||
def inspect_ssh_mux_opts():
|
||||
"""Inspect whether or not ssh supports multiplexing options.
|
||||
|
||||
Return string containing multiplexing options if supported.
|
||||
If ControlPath is supported then placeholder for that path is
|
||||
specified and can be used for final string formatting.
|
||||
For example, this function can return string:
|
||||
"-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
|
||||
Then it can be formatted:
|
||||
mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
|
||||
"""
|
||||
import subprocess
|
||||
|
||||
wanted_mux_opts = {
|
||||
"ControlPath": "{}",
|
||||
"ControlMaster": "auto",
|
||||
"ControlPersist": "125",
|
||||
}
|
||||
mux_opts = " ".join([" -o {}={}".format(
|
||||
x, wanted_mux_opts[x]) for x in wanted_mux_opts])
|
||||
try:
|
||||
subprocess.check_output("ssh {}".format(mux_opts),
|
||||
stderr=subprocess.STDOUT, shell=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
subproc_output = e.output.decode().lower()
|
||||
if "bad configuration option" in subproc_output:
|
||||
return ""
|
||||
return mux_opts
|
||||
|
|
@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC
|
|||
|
||||
CDIST_REMOTE_COPY
|
||||
Use this command for remote copy (should behave like scp).
|
||||
|
||||
CDIST_BETA
|
||||
Enable beta functionalities.
|
||||
eof
|
||||
|
|
|
|||
|
|
@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC
|
|||
CDIST_REMOTE_COPY
|
||||
Use this command for remote copy (should behave like scp).
|
||||
|
||||
CDIST_BETA
|
||||
Enable beta functionalities.
|
||||
|
||||
EXIT STATUS
|
||||
-----------
|
||||
The following exit values shall be returned:
|
||||
|
|
|
|||
176
scripts/cdist
176
scripts/cdist
|
|
@ -1,7 +1,7 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org)
|
||||
# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
|
|
@ -24,182 +24,25 @@
|
|||
import collections
|
||||
import logging
|
||||
|
||||
# list of beta sub-commands
|
||||
BETA_COMMANDS = ['install', ]
|
||||
# list of beta arguments for sub-commands
|
||||
BETA_ARGS = {
|
||||
'config': ['jobs', ],
|
||||
}
|
||||
|
||||
|
||||
def check_positive_int(value):
|
||||
import argparse
|
||||
|
||||
try:
|
||||
val = int(value)
|
||||
except ValueError as e:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{} is invalid int value".format(value))
|
||||
if val <= 0:
|
||||
raise argparse.ArgumentTypeError(
|
||||
"{} is invalid positive int value".format(val))
|
||||
return val
|
||||
|
||||
|
||||
def check_beta(args_dict):
|
||||
if 'beta' not in args_dict:
|
||||
args_dict['beta'] = False
|
||||
# Check only if beta is not enabled: if beta option is specified then
|
||||
# raise error.
|
||||
if not args_dict['beta']:
|
||||
cmd = args_dict['command']
|
||||
# first check if command is beta
|
||||
if cmd in BETA_COMMANDS:
|
||||
raise cdist.CdistBetaRequired(cmd)
|
||||
# then check if command's argument is beta
|
||||
if cmd in BETA_ARGS:
|
||||
for arg in BETA_ARGS[cmd]:
|
||||
if arg in args_dict and args_dict[arg]:
|
||||
raise cdist.CdistBetaRequired(cmd, arg)
|
||||
|
||||
|
||||
_verbosity_level = {
|
||||
0: logging.ERROR,
|
||||
1: logging.WARNING,
|
||||
2: logging.INFO,
|
||||
}
|
||||
_verbosity_level = collections.defaultdict(
|
||||
lambda: logging.DEBUG, _verbosity_level)
|
||||
|
||||
|
||||
def commandline():
|
||||
"""Parse command line"""
|
||||
import argparse
|
||||
|
||||
import cdist.argparse
|
||||
import cdist.banner
|
||||
import cdist.config
|
||||
import cdist.install
|
||||
import cdist.shell
|
||||
import shutil
|
||||
import os
|
||||
import multiprocessing
|
||||
|
||||
# Construct parser others can reuse
|
||||
parser = {}
|
||||
# Options _all_ parsers have in common
|
||||
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['loglevel'].add_argument(
|
||||
'-d', '--debug',
|
||||
help=('Set log level to debug (deprecated, use -vvv instead)'),
|
||||
action='store_true', default=False)
|
||||
parser['loglevel'].add_argument(
|
||||
'-v', '--verbose',
|
||||
help=('Increase log level, be more verbose. Use it more than once '
|
||||
'to increase log level. The order of levels from the lowest '
|
||||
'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
|
||||
action='count', default=0)
|
||||
|
||||
# Main subcommand parser
|
||||
parser['main'] = argparse.ArgumentParser(
|
||||
description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
|
||||
parser['main'].add_argument(
|
||||
'-V', '--version', help='Show version', action='version',
|
||||
version='%(prog)s ' + cdist.VERSION)
|
||||
parser['sub'] = parser['main'].add_subparsers(
|
||||
title="Commands", dest="command")
|
||||
|
||||
# Banner
|
||||
parser['banner'] = parser['sub'].add_parser(
|
||||
'banner', parents=[parser['loglevel']])
|
||||
parser['banner'].set_defaults(func=cdist.banner.banner)
|
||||
|
||||
# Config
|
||||
parser['config'] = parser['sub'].add_parser(
|
||||
'config', parents=[parser['loglevel']])
|
||||
parser['config'].add_argument(
|
||||
'host', nargs='*', help='host(s) to operate on')
|
||||
parser['config'].add_argument(
|
||||
'-b', '--enable-beta',
|
||||
help=('Enable beta functionalities. Beta functionalities '
|
||||
'include the following options: -j/--jobs.'),
|
||||
action='store_true', dest='beta', default=False)
|
||||
parser['config'].add_argument(
|
||||
'-c', '--conf-dir',
|
||||
help=('Add configuration directory (can be repeated, '
|
||||
'last one wins)'), action='append')
|
||||
parser['config'].add_argument(
|
||||
'-f', '--file',
|
||||
help=('Read additional hosts to operate on from specified file '
|
||||
'or from stdin if \'-\' (each host on separate line). '
|
||||
'If no host or host file is specified then, by default, '
|
||||
'read hosts from stdin.'),
|
||||
dest='hostfile', required=False)
|
||||
parser['config'].add_argument(
|
||||
'-i', '--initial-manifest',
|
||||
help='Path to a cdist manifest or \'-\' to read from stdin.',
|
||||
dest='manifest', required=False)
|
||||
parser['config'].add_argument(
|
||||
'-j', '--jobs', nargs='?', type=check_positive_int,
|
||||
help=('Specify the maximum number of parallel jobs, currently '
|
||||
'only global explorers are supported (currently in beta'),
|
||||
action='store', dest='jobs',
|
||||
const=multiprocessing.cpu_count())
|
||||
parser['config'].add_argument(
|
||||
'-n', '--dry-run',
|
||||
help='Do not execute code', action='store_true')
|
||||
parser['config'].add_argument(
|
||||
'-o', '--out-dir',
|
||||
help='Directory to save cdist output in', dest="out_path")
|
||||
parser['config'].add_argument(
|
||||
'-p', '--parallel',
|
||||
help='Operate on multiple hosts in parallel',
|
||||
action='store_true', dest='parallel')
|
||||
parser['config'].add_argument(
|
||||
'-s', '--sequential',
|
||||
help='Operate on multiple hosts sequentially (default)',
|
||||
action='store_false', dest='parallel')
|
||||
# remote-copy and remote-exec defaults are environment variables
|
||||
# if set; if not then None - these will be futher handled after
|
||||
# parsing to determine implementation default
|
||||
parser['config'].add_argument(
|
||||
'--remote-copy',
|
||||
help='Command to use for remote copy (should behave like scp)',
|
||||
action='store', dest='remote_copy',
|
||||
default=os.environ.get('CDIST_REMOTE_COPY'))
|
||||
parser['config'].add_argument(
|
||||
'--remote-exec',
|
||||
help=('Command to use for remote execution '
|
||||
'(should behave like ssh)'),
|
||||
action='store', dest='remote_exec',
|
||||
default=os.environ.get('CDIST_REMOTE_EXEC'))
|
||||
parser['config'].set_defaults(func=cdist.config.Config.commandline)
|
||||
|
||||
# Install
|
||||
parser['install'] = parser['sub'].add_parser('install', add_help=False,
|
||||
parents=[parser['config']])
|
||||
parser['install'].set_defaults(func=cdist.install.Install.commandline)
|
||||
|
||||
# Shell
|
||||
parser['shell'] = parser['sub'].add_parser(
|
||||
'shell', parents=[parser['loglevel']])
|
||||
parser['shell'].add_argument(
|
||||
'-s', '--shell',
|
||||
help=('Select shell to use, defaults to current shell. Used shell'
|
||||
' should be POSIX compatible shell.'))
|
||||
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
|
||||
|
||||
for p in parser:
|
||||
parser[p].epilog = (
|
||||
"Get cdist at http://www.nico.schottelius.org/software/cdist/")
|
||||
|
||||
parser = cdist.argparse.get_parsers()
|
||||
args = parser['main'].parse_args(sys.argv[1:])
|
||||
|
||||
# Loglevels are handled globally in here
|
||||
if args.debug:
|
||||
log.warning("-d/--debug is deprecated, use -vvv instead")
|
||||
args.verbose = 3
|
||||
|
||||
logging.root.setLevel(_verbosity_level[args.verbose])
|
||||
retval = cdist.argparse.handle_loglevel(args)
|
||||
if retval:
|
||||
log.warning(retval)
|
||||
|
||||
log.debug(args)
|
||||
log.info("version %s" % cdist.VERSION)
|
||||
|
|
@ -219,17 +62,16 @@ def commandline():
|
|||
parser['main'].print_help()
|
||||
sys.exit(0)
|
||||
|
||||
check_beta(vars(args))
|
||||
cdist.argparse.check_beta(vars(args))
|
||||
args.func(args)
|
||||
|
||||
if __name__ == "__main__":
|
||||
# Sys is needed for sys.exit()
|
||||
import sys
|
||||
|
||||
cdistpythonversion = '3.2'
|
||||
if sys.version < cdistpythonversion:
|
||||
print('Python >= ' + cdistpythonversion +
|
||||
' is required on the source host.', file=sys.stderr)
|
||||
print('Python >= {} is required on the source host.'.format(
|
||||
cdistpythonversion), file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
exit_code = 0
|
||||
|
|
|
|||
Loading…
Reference in New Issue