Merge pull request #504 from darko-poljak/the-good-the-bad-and-the-ugly

The Good, the Bad and the Ugly
This commit is contained in:
Darko Poljak 2016-12-07 18:45:56 +01:00 committed by GitHub
commit bc5f6c8923
12 changed files with 528 additions and 374 deletions

View file

@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error):
err_msg = ("\'{}\' command is beta, but beta is " err_msg = ("\'{}\' command is beta, but beta is "
"not enabled. If you want to use it please enable beta " "not enabled. If you want to use it please enable beta "
"functionalities by using the -b/--enable-beta command " "functionalities by using the -b/--enable-beta command "
"line flag.") "line flag or setting CDIST_BETA env var.")
fmt_args = [self.command, ] fmt_args = [self.command, ]
else: else:
err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta " err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta "
"is not enabled. If you want to use it please enable " "is not enabled. If you want to use it please enable "
"beta functionalities by using the -b/--enable-beta " "beta functionalities by using the -b/--enable-beta "
"command line flag.") "command line flag or setting CDIST_BETA env var.")
fmt_args = [self.arg, self.command, ] fmt_args = [self.arg, self.command, ]
return err_msg.format(*fmt_args) return err_msg.format(*fmt_args)

208
cdist/argparse.py Normal file
View file

@ -0,0 +1,208 @@
import argparse
import cdist
import multiprocessing
import os
import logging
import collections
# set of beta sub-commands
BETA_COMMANDS = set(('install', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
'config': set(('jobs', )),
}
EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
# Parser others can reuse
parser = None
_verbosity_level = {
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
}
_verbosity_level = collections.defaultdict(
lambda: logging.DEBUG, _verbosity_level)
def add_beta_command(cmd):
BETA_COMMANDS.add(cmd)
def add_beta_arg(cmd, arg):
if cmd in BETA_ARGS:
if arg not in BETA_ARGS[cmd]:
BETA_ARGS[cmd].append(arg)
else:
BETA_ARGS[cmd] = set((arg, ))
def check_beta(args_dict):
if 'beta' not in args_dict:
args_dict['beta'] = False
# Check only if beta is not enabled: if beta option is specified then
# raise error.
if not args_dict['beta']:
cmd = args_dict['command']
# first check if command is beta
if cmd in BETA_COMMANDS:
raise cdist.CdistBetaRequired(cmd)
# then check if some command's argument is beta
if cmd in BETA_ARGS:
for arg in BETA_ARGS[cmd]:
if arg in args_dict and args_dict[arg]:
raise cdist.CdistBetaRequired(cmd, arg)
def check_positive_int(value):
import argparse
try:
val = int(value)
except ValueError:
raise argparse.ArgumentTypeError(
"{} is invalid int value".format(value))
if val <= 0:
raise argparse.ArgumentTypeError(
"{} is invalid positive int value".format(val))
return val
def get_parsers():
global parser
# Construct parser others can reuse
if parser:
return parser
else:
parser = {}
# Options _all_ parsers have in common
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['loglevel'].add_argument(
'-d', '--debug',
help=('Set log level to debug (deprecated, use -vvv instead)'),
action='store_true', default=False)
parser['loglevel'].add_argument(
'-v', '--verbose',
help=('Increase log level, be more verbose. Use it more than once '
'to increase log level. The order of levels from the lowest '
'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
action='count', default=0)
parser['beta'] = argparse.ArgumentParser(add_help=False)
parser['beta'].add_argument(
'-b', '--beta',
help=('Enable beta functionalities. '
'Can also be enabled using CDIST_BETA env var.'),
action='store_true', dest='beta',
default='CDIST_BETA' in os.environ)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(
description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
parser['main'].add_argument(
'-V', '--version', help='Show version', action='version',
version='%(prog)s ' + cdist.VERSION)
parser['sub'] = parser['main'].add_subparsers(
title="Commands", dest="command")
# Banner
parser['banner'] = parser['sub'].add_parser(
'banner', parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config
parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
'-c', '--conf-dir',
help=('Add configuration directory (can be repeated, '
'last one wins)'), action='append')
parser['config_main'].add_argument(
'-i', '--initial-manifest',
help='path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['config_main'].add_argument(
'-j', '--jobs', nargs='?',
type=check_positive_int,
help=('Specify the maximum number of parallel jobs, currently '
'only global explorers are supported'),
action='store', dest='jobs',
const=multiprocessing.cpu_count())
parser['config_main'].add_argument(
'-n', '--dry-run',
help='do not execute code', action='store_true')
parser['config_main'].add_argument(
'-o', '--out-dir',
help='directory to save cdist output in', dest="out_path")
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config_main'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
parser['config_main'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh)'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
# Config
parser['config_args'] = argparse.ArgumentParser(add_help=False)
parser['config_args'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config_args'].add_argument(
'-f', '--file',
help=('Read additional hosts to operate on from specified file '
'or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'read hosts from stdin.'),
dest='hostfile', required=False)
parser['config_args'].add_argument(
'-p', '--parallel',
help='operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config_args'].add_argument(
'-s', '--sequential',
help='operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel'], parser['beta'],
parser['config_main'],
parser['config_args']])
parser['config'].set_defaults(func=cdist.config.Config.commandline)
# Install
parser['install'] = parser['sub'].add_parser('install', add_help=False,
parents=[parser['config']])
parser['install'].set_defaults(func=cdist.install.Install.commandline)
# Shell
parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']])
parser['shell'].add_argument(
'-s', '--shell',
help=('Select shell to use, defaults to current shell. Used shell'
' should be POSIX compatible shell.'))
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
for p in parser:
parser[p].epilog = EPILOG
return parser
def handle_loglevel(args):
if args.debug:
retval = "-d/--debug is deprecated, use -vvv instead"
args.verbose = 3
else:
retval = None
logging.root.setLevel(_verbosity_level[args.verbose])
return retval

View file

@ -22,50 +22,21 @@
import logging import logging
import os import os
import shutil
import sys import sys
import time import time
import pprint
import itertools import itertools
import tempfile import tempfile
import socket import socket
import cdist import cdist
import cdist.hostsource
import cdist.exec.local import cdist.exec.local
import cdist.exec.remote import cdist.exec.remote
import cdist.util.ipaddr as ipaddr
from cdist import core from cdist import core
from cdist.util.remoteutil import inspect_ssh_mux_opts
def inspect_ssh_mux_opts():
"""Inspect whether or not ssh supports multiplexing options.
Return string containing multiplexing options if supported.
If ControlPath is supported then placeholder for that path is
specified and can be used for final string formatting.
For example, this function can return string:
"-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
Then it can be formatted:
mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
"""
import subprocess
wanted_mux_opts = {
"ControlPath": "{}",
"ControlMaster": "auto",
"ControlPersist": "125",
}
mux_opts = " ".join([" -o {}={}".format(
x, wanted_mux_opts[x]) for x in wanted_mux_opts])
try:
subprocess.check_output("ssh {}".format(mux_opts),
stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
subproc_output = e.output.decode().lower()
if "bad configuration option" in subproc_output:
return ""
return mux_opts
class Config(object): class Config(object):
@ -89,55 +60,17 @@ class Config(object):
self.local.create_files_dirs() self.local.create_files_dirs()
self.remote.create_files_dirs() self.remote.create_files_dirs()
@staticmethod
def hostfile_process_line(line):
"""Return host from read line or None if no host present."""
if not line:
return None
# remove comment if present
comment_index = line.find('#')
if comment_index >= 0:
host = line[:comment_index]
else:
host = line
# remove leading and trailing whitespaces
host = host.strip()
# skip empty lines
if host:
return host
else:
return None
@staticmethod @staticmethod
def hosts(source): def hosts(source):
"""Yield hosts from source. try:
Source can be a sequence or filename (stdin if \'-\'). yield from cdist.hostsource.HostSource(source)()
In case of filename each line represents one host. except (IOError, OSError, UnicodeError) as e:
""" raise cdist.Error(
if isinstance(source, str): "Error reading hosts from \'{}\': {}".format(
import fileinput source, e))
try:
for host in fileinput.input(files=(source)):
host = Config.hostfile_process_line(host)
if host:
yield host
except (IOError, OSError, UnicodeError) as e:
raise cdist.Error(
"Error reading hosts from file \'{}\': {}".format(
source, e))
else:
if source:
for host in source:
yield host
@classmethod @classmethod
def commandline(cls, args): def _check_and_prepare_args(cls, args):
"""Configure remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
if args.manifest == '-' and args.hostfile == '-': if args.manifest == '-' and args.hostfile == '-':
raise cdist.Error(("Cannot read both, manifest and host file, " raise cdist.Error(("Cannot read both, manifest and host file, "
"from stdin")) "from stdin"))
@ -162,10 +95,6 @@ class Config(object):
import atexit import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path)) atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
# default remote cmd patterns # default remote cmd patterns
args.remote_exec_pattern = None args.remote_exec_pattern = None
args.remote_copy_pattern = None args.remote_copy_pattern = None
@ -182,10 +111,29 @@ class Config(object):
if args_dict['remote_copy'] is None: if args_dict['remote_copy'] is None:
args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
@classmethod
def _base_root_path(cls, args):
if args.out_path: if args.out_path:
base_root_path = args.out_path base_root_path = args.out_path
else: else:
base_root_path = tempfile.mkdtemp() base_root_path = tempfile.mkdtemp()
return base_root_path
@classmethod
def commandline(cls, args):
"""Configure remote system"""
import multiprocessing
# FIXME: Refactor relict - remove later
log = logging.getLogger("cdist")
cls._check_and_prepare_args(args)
process = {}
failed_hosts = []
time_start = time.time()
base_root_path = cls._base_root_path(args)
hostcnt = 0 hostcnt = 0
for host in itertools.chain(cls.hosts(args.host), for host in itertools.chain(cls.hosts(args.host),
@ -227,6 +175,24 @@ class Config(object):
raise cdist.Error("Failed to configure the following hosts: " + raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts)) " ".join(failed_hosts))
@classmethod
def _resolve_remote_cmds(cls, args, host_base_path):
control_path = os.path.join(host_base_path, "ssh-control-path")
# If we constructed patterns for remote commands then there is
# placeholder for ssh ControlPath, format it and we have unique
# ControlPath for each host.
#
# If not then use args.remote_exec/copy that user specified.
if args.remote_exec_pattern:
remote_exec = args.remote_exec_pattern.format(control_path)
else:
remote_exec = args.remote_exec
if args.remote_copy_pattern:
remote_copy = args.remote_copy_pattern.format(control_path)
else:
remote_copy = args.remote_copy
return (remote_exec, remote_copy, )
@classmethod @classmethod
def onehost(cls, host, host_base_path, host_dir_name, args, parallel): def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
"""Configure ONE system""" """Configure ONE system"""
@ -234,57 +200,14 @@ class Config(object):
log = logging.getLogger(host) log = logging.getLogger(host)
try: try:
control_path = os.path.join(host_base_path, "ssh-control-path") remote_exec, remote_copy = cls._resolve_remote_cmds(
# If we constructed patterns for remote commands then there is args, host_base_path)
# placeholder for ssh ControlPath, format it and we have unique
# ControlPath for each host.
#
# If not then use args.remote_exec/copy that user specified.
if args.remote_exec_pattern:
remote_exec = args.remote_exec_pattern.format(control_path)
else:
remote_exec = args.remote_exec
if args.remote_copy_pattern:
remote_copy = args.remote_copy_pattern.format(control_path)
else:
remote_copy = args.remote_copy
log.debug("remote_exec for host \"{}\": {}".format( log.debug("remote_exec for host \"{}\": {}".format(
host, remote_exec)) host, remote_exec))
log.debug("remote_copy for host \"{}\": {}".format( log.debug("remote_copy for host \"{}\": {}".format(
host, remote_copy)) host, remote_copy))
try: target_host = ipaddr.resolve_target_addresses(host)
# getaddrinfo returns a list of 5-tuples:
# (family, type, proto, canonname, sockaddr)
# where sockaddr is:
# (address, port) for AF_INET,
# (address, port, flow_info, scopeid) for AF_INET6
ip_addr = socket.getaddrinfo(
host, None, type=socket.SOCK_STREAM)[0][4][0]
# gethostbyaddr returns triple
# (hostname, aliaslist, ipaddrlist)
host_name = socket.gethostbyaddr(ip_addr)[0]
log.debug("derived host_name for host \"{}\": {}".format(
host, host_name))
except (socket.gaierror, socket.herror) as e:
log.warn("Could not derive host_name for {}"
", $host_name will be empty. Error is: {}".format(
host, e))
# in case of error provide empty value
host_name = ''
try:
host_fqdn = socket.getfqdn(host)
log.debug("derived host_fqdn for host \"{}\": {}".format(
host, host_fqdn))
except socket.herror as e:
log.warn("Could not derive host_fqdn for {}"
", $host_fqdn will be empty. Error is: {}".format(
host, e))
# in case of error provide empty value
host_fqdn = ''
target_host = (host, host_name, host_fqdn)
log.debug("target_host: {}".format(target_host)) log.debug("target_host: {}".format(target_host))
local = cdist.exec.local.Local( local = cdist.exec.local.Local(

View file

@ -149,14 +149,9 @@ class Explorer(object):
def transfer_global_explorers(self): def transfer_global_explorers(self):
"""Transfer the global explorers to the remote side.""" """Transfer the global explorers to the remote side."""
self.remote.mkdir(self.remote.global_explorer_path) self.remote.mkdir(self.remote.global_explorer_path)
if self.jobs is None: self.remote.transfer(self.local.global_explorer_path,
self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path,
self.remote.global_explorer_path) self.jobs)
else:
self.remote.transfer_dir_parallel(
self.local.global_explorer_path,
self.remote.global_explorer_path,
self.jobs)
self.remote.run(["chmod", "0700", self.remote.run(["chmod", "0700",
"%s/*" % (self.remote.global_explorer_path)]) "%s/*" % (self.remote.global_explorer_path)])

View file

@ -30,40 +30,13 @@ import multiprocessing
import cdist import cdist
import cdist.exec.util as exec_util import cdist.exec.util as exec_util
import cdist.util.ipaddr as ipaddr
# check whether addr is IPv6
try:
# python 3.3+
import ipaddress
def _is_ipv6(addr):
try:
return ipaddress.ip_address(addr).version == 6
except ValueError:
return False
except ImportError:
# fallback for older python versions
import socket
def _is_ipv6(addr):
try:
socket.inet_aton(addr)
return False
except socket.error:
pass
try:
socket.inet_pton(socket.AF_INET6, addr)
return True
except socket.error:
pass
return False
def _wrap_addr(addr): def _wrap_addr(addr):
"""If addr is IPv6 then return addr wrapped between '[' and ']', """If addr is IPv6 then return addr wrapped between '[' and ']',
otherwise return it intact.""" otherwise return it intact."""
if _is_ipv6(addr): if ipaddr.is_ipv6(addr):
return "".join(("[", addr, "]", )) return "".join(("[", addr, "]", ))
else: else:
return addr return addr
@ -145,57 +118,59 @@ class Remote(object):
self.log.debug("Remote mkdir: %s", path) self.log.debug("Remote mkdir: %s", path)
self.run(["mkdir", "-p", path]) self.run(["mkdir", "-p", path])
def transfer(self, source, destination): def transfer(self, source, destination, jobs=None):
"""Transfer a file or directory to the remote side.""" """Transfer a file or directory to the remote side."""
self.log.debug("Remote transfer: %s -> %s", source, destination) self.log.debug("Remote transfer: %s -> %s", source, destination)
self.rmdir(destination) self.rmdir(destination)
if os.path.isdir(source): if os.path.isdir(source):
self.mkdir(destination) self.mkdir(destination)
for f in glob.glob1(source, '*'): if jobs:
command = self._copy.split() self._transfer_dir_parallel(source, destination, jobs)
path = os.path.join(source, f) else:
command.extend([path, '{0}:{1}'.format( self._transfer_dir_sequential(source, destination)
_wrap_addr(self.target_host[0]), destination)]) elif jobs:
self._run_command(command) raise cdist.Error("Source {} is not a directory".format(source))
else: else:
command = self._copy.split() command = self._copy.split()
command.extend([source, '{0}:{1}'.format( command.extend([source, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)]) _wrap_addr(self.target_host[0]), destination)])
self._run_command(command) self._run_command(command)
def transfer_dir_parallel(self, source, destination, jobs): def _transfer_dir_sequential(self, source, destination):
"""Transfer a directory to the remote side in parallel mode.""" for f in glob.glob1(source, '*'):
self.log.debug("Remote transfer: %s -> %s", source, destination) command = self._copy.split()
self.rmdir(destination) path = os.path.join(source, f)
if os.path.isdir(source): command.extend([path, '{0}:{1}'.format(
self.mkdir(destination) _wrap_addr(self.target_host[0]), destination)])
self.log.info("Remote transfer in {} parallel jobs".format( self._run_command(command)
jobs))
self.log.debug("Multiprocessing start method is {}".format(
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for parallel "
"remote transfer"))
with multiprocessing.Pool(jobs) as pool:
self.log.debug("Starting async for parallel transfer")
commands = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
commands.append(command)
results = [
pool.apply_async(self._run_command, (cmd,))
for cmd in commands
]
self.log.debug("Waiting async results for parallel transfer") def _transfer_dir_parallel(self, source, destination, jobs):
for r in results: """Transfer a directory to the remote side in parallel mode."""
r.get() # self._run_command returns None self.log.info("Remote transfer in {} parallel jobs".format(
self.log.debug(("Multiprocessing for parallel transfer " jobs))
"finished")) self.log.debug("Multiprocessing start method is {}".format(
else: multiprocessing.get_start_method()))
raise cdist.Error("Source {} is not a directory".format(source)) self.log.debug(("Starting multiprocessing Pool for parallel "
"remote transfer"))
with multiprocessing.Pool(jobs) as pool:
self.log.debug("Starting async for parallel transfer")
commands = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
commands.append(command)
results = [
pool.apply_async(self._run_command, (cmd,))
for cmd in commands
]
self.log.debug("Waiting async results for parallel transfer")
for r in results:
r.get() # self._run_command returns None
self.log.debug(("Multiprocessing for parallel transfer "
"finished"))
def run_script(self, script, env=None, return_output=False): def run_script(self, script, env=None, return_output=False):
"""Run the given script with the given environment on the remote side. """Run the given script with the given environment on the remote side.

72
cdist/hostsource.py Normal file
View file

@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import fileinput
class HostSource(object):
"""
Host source object.
Source can be a sequence or filename (stdin if \'-\').
In case of filename each line represents one host.
"""
def __init__(self, source):
self.source = source
def _process_file_line(self, line):
"""Return host from read line or None if no host present."""
if not line:
return None
# remove comment if present
comment_index = line.find('#')
if comment_index >= 0:
host = line[:comment_index]
else:
host = line
# remove leading and trailing whitespaces
host = host.strip()
# skip empty lines
if host:
return host
else:
return None
def _hosts_from_sequence(self):
for host in self.source:
yield host
def _hosts_from_file(self):
for line in fileinput.input(files=(self.source)):
host = self._process_file_line(line)
if host:
yield host
def hosts(self):
if not self.source:
return
if isinstance(self.source, str):
yield from self._hosts_from_file()
else:
yield from self._hosts_from_sequence()
def __call__(self):
yield from self.hosts()

View file

@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase):
source_file_name = os.path.split(source_file)[-1] source_file_name = os.path.split(source_file)[-1]
filenames.append(source_file_name) filenames.append(source_file_name)
target = self.mkdtemp(dir=self.temp_dir) target = self.mkdtemp(dir=self.temp_dir)
self.remote.transfer_dir_parallel(source, target, self.remote.transfer(source, target,
multiprocessing.cpu_count()) multiprocessing.cpu_count())
# test if the payload files are in the target directory # test if the payload files are in the target directory
for filename in filenames: for filename in filenames:
self.assertTrue(os.path.isfile(os.path.join(target, filename))) self.assertTrue(os.path.isfile(os.path.join(target, filename)))

83
cdist/util/ipaddr.py Normal file
View file

@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import socket
import logging
def resolve_target_addresses(host):
log = logging.getLogger(host)
try:
# getaddrinfo returns a list of 5-tuples:
# (family, type, proto, canonname, sockaddr)
# where sockaddr is:
# (address, port) for AF_INET,
# (address, port, flow_info, scopeid) for AF_INET6
ip_addr = socket.getaddrinfo(
host, None, type=socket.SOCK_STREAM)[0][4][0]
# gethostbyaddr returns triple
# (hostname, aliaslist, ipaddrlist)
host_name = socket.gethostbyaddr(ip_addr)[0]
log.debug("derived host_name for host \"{}\": {}".format(
host, host_name))
except (socket.gaierror, socket.herror) as e:
log.warn("Could not derive host_name for {}"
", $host_name will be empty. Error is: {}".format(host, e))
# in case of error provide empty value
host_name = ''
try:
host_fqdn = socket.getfqdn(host)
log.debug("derived host_fqdn for host \"{}\": {}".format(
host, host_fqdn))
except socket.herror as e:
log.warn("Could not derive host_fqdn for {}"
", $host_fqdn will be empty. Error is: {}".format(host, e))
# in case of error provide empty value
host_fqdn = ''
return (host, host_name, host_fqdn)
# check whether addr is IPv6
try:
# python 3.3+
import ipaddress
def is_ipv6(addr):
try:
return ipaddress.ip_address(addr).version == 6
except ValueError:
return False
except ImportError:
# fallback for older python versions
def is_ipv6(addr):
try:
socket.inet_aton(addr)
return False
except socket.error:
pass
try:
socket.inet_pton(socket.AF_INET6, addr)
return True
except socket.error:
pass
return False

50
cdist/util/remoteutil.py Normal file
View file

@ -0,0 +1,50 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
def inspect_ssh_mux_opts():
"""Inspect whether or not ssh supports multiplexing options.
Return string containing multiplexing options if supported.
If ControlPath is supported then placeholder for that path is
specified and can be used for final string formatting.
For example, this function can return string:
"-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
Then it can be formatted:
mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
"""
import subprocess
wanted_mux_opts = {
"ControlPath": "{}",
"ControlMaster": "auto",
"ControlPersist": "125",
}
mux_opts = " ".join([" -o {}={}".format(
x, wanted_mux_opts[x]) for x in wanted_mux_opts])
try:
subprocess.check_output("ssh {}".format(mux_opts),
stderr=subprocess.STDOUT, shell=True)
except subprocess.CalledProcessError as e:
subproc_output = e.output.decode().lower()
if "bad configuration option" in subproc_output:
return ""
return mux_opts

View file

@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp). Use this command for remote copy (should behave like scp).
CDIST_BETA
Enable beta functionalities.
eof eof

View file

@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC
CDIST_REMOTE_COPY CDIST_REMOTE_COPY
Use this command for remote copy (should behave like scp). Use this command for remote copy (should behave like scp).
CDIST_BETA
Enable beta functionalities.
EXIT STATUS EXIT STATUS
----------- -----------
The following exit values shall be returned: The following exit values shall be returned:

View file

@ -1,7 +1,7 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2016 Nico Schottelius (nico-cdist at schottelius.org)
# 2016 Darko Poljak (darko.poljak at gmail.com) # 2016 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
@ -24,182 +24,25 @@
import collections import collections
import logging import logging
# list of beta sub-commands
BETA_COMMANDS = ['install', ]
# list of beta arguments for sub-commands
BETA_ARGS = {
'config': ['jobs', ],
}
def check_positive_int(value):
import argparse
try:
val = int(value)
except ValueError as e:
raise argparse.ArgumentTypeError(
"{} is invalid int value".format(value))
if val <= 0:
raise argparse.ArgumentTypeError(
"{} is invalid positive int value".format(val))
return val
def check_beta(args_dict):
if 'beta' not in args_dict:
args_dict['beta'] = False
# Check only if beta is not enabled: if beta option is specified then
# raise error.
if not args_dict['beta']:
cmd = args_dict['command']
# first check if command is beta
if cmd in BETA_COMMANDS:
raise cdist.CdistBetaRequired(cmd)
# then check if command's argument is beta
if cmd in BETA_ARGS:
for arg in BETA_ARGS[cmd]:
if arg in args_dict and args_dict[arg]:
raise cdist.CdistBetaRequired(cmd, arg)
_verbosity_level = {
0: logging.ERROR,
1: logging.WARNING,
2: logging.INFO,
}
_verbosity_level = collections.defaultdict(
lambda: logging.DEBUG, _verbosity_level)
def commandline(): def commandline():
"""Parse command line""" """Parse command line"""
import argparse
import cdist.argparse
import cdist.banner import cdist.banner
import cdist.config import cdist.config
import cdist.install import cdist.install
import cdist.shell import cdist.shell
import shutil import shutil
import os import os
import multiprocessing
# Construct parser others can reuse
parser = {}
# Options _all_ parsers have in common
parser['loglevel'] = argparse.ArgumentParser(add_help=False)
parser['loglevel'].add_argument(
'-d', '--debug',
help=('Set log level to debug (deprecated, use -vvv instead)'),
action='store_true', default=False)
parser['loglevel'].add_argument(
'-v', '--verbose',
help=('Increase log level, be more verbose. Use it more than once '
'to increase log level. The order of levels from the lowest '
'to the highest are: ERROR, WARNING, INFO, DEBUG.'),
action='count', default=0)
# Main subcommand parser
parser['main'] = argparse.ArgumentParser(
description='cdist ' + cdist.VERSION, parents=[parser['loglevel']])
parser['main'].add_argument(
'-V', '--version', help='Show version', action='version',
version='%(prog)s ' + cdist.VERSION)
parser['sub'] = parser['main'].add_subparsers(
title="Commands", dest="command")
# Banner
parser['banner'] = parser['sub'].add_parser(
'banner', parents=[parser['loglevel']])
parser['banner'].set_defaults(func=cdist.banner.banner)
# Config
parser['config'] = parser['sub'].add_parser(
'config', parents=[parser['loglevel']])
parser['config'].add_argument(
'host', nargs='*', help='host(s) to operate on')
parser['config'].add_argument(
'-b', '--enable-beta',
help=('Enable beta functionalities. Beta functionalities '
'include the following options: -j/--jobs.'),
action='store_true', dest='beta', default=False)
parser['config'].add_argument(
'-c', '--conf-dir',
help=('Add configuration directory (can be repeated, '
'last one wins)'), action='append')
parser['config'].add_argument(
'-f', '--file',
help=('Read additional hosts to operate on from specified file '
'or from stdin if \'-\' (each host on separate line). '
'If no host or host file is specified then, by default, '
'read hosts from stdin.'),
dest='hostfile', required=False)
parser['config'].add_argument(
'-i', '--initial-manifest',
help='Path to a cdist manifest or \'-\' to read from stdin.',
dest='manifest', required=False)
parser['config'].add_argument(
'-j', '--jobs', nargs='?', type=check_positive_int,
help=('Specify the maximum number of parallel jobs, currently '
'only global explorers are supported (currently in beta'),
action='store', dest='jobs',
const=multiprocessing.cpu_count())
parser['config'].add_argument(
'-n', '--dry-run',
help='Do not execute code', action='store_true')
parser['config'].add_argument(
'-o', '--out-dir',
help='Directory to save cdist output in', dest="out_path")
parser['config'].add_argument(
'-p', '--parallel',
help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel')
parser['config'].add_argument(
'-s', '--sequential',
help='Operate on multiple hosts sequentially (default)',
action='store_false', dest='parallel')
# remote-copy and remote-exec defaults are environment variables
# if set; if not then None - these will be futher handled after
# parsing to determine implementation default
parser['config'].add_argument(
'--remote-copy',
help='Command to use for remote copy (should behave like scp)',
action='store', dest='remote_copy',
default=os.environ.get('CDIST_REMOTE_COPY'))
parser['config'].add_argument(
'--remote-exec',
help=('Command to use for remote execution '
'(should behave like ssh)'),
action='store', dest='remote_exec',
default=os.environ.get('CDIST_REMOTE_EXEC'))
parser['config'].set_defaults(func=cdist.config.Config.commandline)
# Install
parser['install'] = parser['sub'].add_parser('install', add_help=False,
parents=[parser['config']])
parser['install'].set_defaults(func=cdist.install.Install.commandline)
# Shell
parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']])
parser['shell'].add_argument(
'-s', '--shell',
help=('Select shell to use, defaults to current shell. Used shell'
' should be POSIX compatible shell.'))
parser['shell'].set_defaults(func=cdist.shell.Shell.commandline)
for p in parser:
parser[p].epilog = (
"Get cdist at http://www.nico.schottelius.org/software/cdist/")
parser = cdist.argparse.get_parsers()
args = parser['main'].parse_args(sys.argv[1:]) args = parser['main'].parse_args(sys.argv[1:])
# Loglevels are handled globally in here # Loglevels are handled globally in here
if args.debug: retval = cdist.argparse.handle_loglevel(args)
log.warning("-d/--debug is deprecated, use -vvv instead") if retval:
args.verbose = 3 log.warning(retval)
logging.root.setLevel(_verbosity_level[args.verbose])
log.debug(args) log.debug(args)
log.info("version %s" % cdist.VERSION) log.info("version %s" % cdist.VERSION)
@ -219,17 +62,16 @@ def commandline():
parser['main'].print_help() parser['main'].print_help()
sys.exit(0) sys.exit(0)
check_beta(vars(args)) cdist.argparse.check_beta(vars(args))
args.func(args) args.func(args)
if __name__ == "__main__": if __name__ == "__main__":
# Sys is needed for sys.exit()
import sys import sys
cdistpythonversion = '3.2' cdistpythonversion = '3.2'
if sys.version < cdistpythonversion: if sys.version < cdistpythonversion:
print('Python >= ' + cdistpythonversion + print('Python >= {} is required on the source host.'.format(
' is required on the source host.', file=sys.stderr) cdistpythonversion), file=sys.stderr)
sys.exit(1) sys.exit(1)
exit_code = 0 exit_code = 0