Save output streams.

This commit is contained in:
Steven Armstrong 2013-09-05 22:01:21 +02:00 committed by Darko Poljak
parent 13a13eee03
commit 00cd5fa91e
19 changed files with 449 additions and 109 deletions

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2012-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -42,7 +43,7 @@ BANNER = """
"P' "" "" "P' "" ""
""" """
REMOTE_COPY = "scp -o User=root" REMOTE_COPY = "scp -o User=root -q"
REMOTE_EXEC = "ssh -o User=root" REMOTE_EXEC = "ssh -o User=root"
REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
@ -81,17 +82,47 @@ class CdistBetaRequired(cdist.Error):
class CdistObjectError(Error): class CdistObjectError(Error):
"""Something went wrong with an object""" """Something went wrong while working on a specific cdist object"""
def __init__(self, cdist_object, subject=''):
self.cdist_object = cdist_object
self.object_name = cdist_object.name.center(len(cdist_object.name)+2)
if isinstance(subject, Error):
self.original_error = subject
else:
self.original_error = None
self.message = str(subject)
self.line_length = 74
def __init__(self, cdist_object, message): @property
self.name = cdist_object.name def stderr(self):
self.source = " ".join(cdist_object.source) output = []
self.message = message for stderr_name in os.listdir(self.cdist_object.stderr_path):
stderr_path = os.path.join(self.cdist_object.stderr_path,
stderr_name)
# label = '---- '+ stderr_name +':stderr '
label = stderr_name + ':stderr '
if os.path.getsize(stderr_path) > 0:
# output.append(label)
# output.append('{0:-^50}'.format(label.center(len(label)+2)))
output.append('{0:-<{1}}'.format(label, self.line_length))
with open(stderr_path, 'r') as fd:
output.append(fd.read())
return '\n'.join(output)
def __str__(self): def __str__(self):
return '%s: %s (defined at %s)' % (self.name, output = []
self.message, output.append(self.message)
self.source) output.append('''{label:-<{length}}
name: {o.name}
path: {o.absolute_path}
source: {o.source}
type: {o.cdist_type.absolute_path}'''.format(
label='---- object ',
length=self.line_length,
o=self.cdist_object)
)
output.append(self.stderr)
return '\n'.join(output)
def file_to_list(filename): def file_to_list(filename):

View file

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2013-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
@ -353,7 +354,9 @@ class Config(object):
base_path=args.remote_out_path, base_path=args.remote_out_path,
quiet_mode=args.quiet, quiet_mode=args.quiet,
archiving_mode=args.use_archiving, archiving_mode=args.use_archiving,
configuration=configuration) configuration=configuration,
stdout_base_path=local.stdout_base_path,
stderr_base_path=local.stderr_base_path)
cleanup_cmds = [] cleanup_cmds = []
if cleanup_cmd: if cleanup_cmd:
@ -453,7 +456,9 @@ class Config(object):
objects_changed = False objects_changed = False
for cdist_object in self.object_list(): for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements): try:
if cdist_object.requirements_unfinished(
cdist_object.requirements):
"""We cannot do anything for this poor object""" """We cannot do anything for this poor object"""
continue continue
@ -463,7 +468,8 @@ class Config(object):
self.object_prepare(cdist_object) self.object_prepare(cdist_object)
objects_changed = True objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire): if cdist_object.requirements_unfinished(
cdist_object.autorequire):
"""The previous step created objects we depend on - """The previous step created objects we depend on -
wait for them wait for them
""" """
@ -472,6 +478,8 @@ class Config(object):
if cdist_object.state == core.CdistObject.STATE_PREPARED: if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object) self.object_run(cdist_object)
objects_changed = True objects_changed = True
except cdist.Error as e:
raise cdist.CdistObjectError(cdist_object, e)
return objects_changed return objects_changed

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2014 Daniel Heule (hda at sfs.biz) # 2014 Daniel Heule (hda at sfs.biz)
# #
@ -80,6 +80,8 @@ class CdistObject(object):
self.code_local_path = os.path.join(self.path, "code-local") self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote") self.code_remote_path = os.path.join(self.path, "code-remote")
self.parameter_path = os.path.join(self.path, "parameter") self.parameter_path = os.path.join(self.path, "parameter")
self.stdout_path = os.path.join(self.absolute_path, "stdout")
self.stderr_path = os.path.join(self.absolute_path, "stderr")
@classmethod @classmethod
def list_objects(cls, object_base_path, type_base_path, object_marker): def list_objects(cls, object_base_path, type_base_path, object_marker):
@ -246,10 +248,11 @@ class CdistObject(object):
"""Create this cdist object on the filesystem. """Create this cdist object on the filesystem.
""" """
try: try:
os.makedirs(self.absolute_path, exist_ok=allow_overwrite) for path in (self.absolute_path,
absolute_parameter_path = os.path.join(self.base_path, os.path.join(self.base_path, self.parameter_path),
self.parameter_path) self.stdout_path,
os.makedirs(absolute_parameter_path, exist_ok=allow_overwrite) self.stderr_path):
os.makedirs(path, exist_ok=allow_overwrite)
except EnvironmentError as error: except EnvironmentError as error:
raise cdist.Error(('Error creating directories for cdist object: ' raise cdist.Error(('Error creating directories for cdist object: '
'%s: %s') % (self, error)) '%s: %s') % (self, error))

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# 2014 Daniel Heule (hda at sfs.biz) # 2014 Daniel Heule (hda at sfs.biz)
# #
@ -127,8 +127,13 @@ class Code(object):
'__object_name': cdist_object.name, '__object_name': cdist_object.name,
}) })
message_prefix = cdist_object.name message_prefix = cdist_object.name
return self.local.run_script(script, env=env, return_output=True, stderr_path = os.path.join(cdist_object.stderr_path,
message_prefix=message_prefix) 'gencode-' + which)
with open(stderr_path, 'ba+') as stderr:
return self.local.run_script(script, env=env,
return_output=True,
message_prefix=message_prefix,
stderr=stderr)
def run_gencode_local(self, cdist_object): def run_gencode_local(self, cdist_object):
"""Run the gencode-local script for the given cdist object.""" """Run the gencode-local script for the given cdist object."""
@ -152,7 +157,11 @@ class Code(object):
which_exec = getattr(self, which) which_exec = getattr(self, which)
script = os.path.join(which_exec.object_path, script = os.path.join(which_exec.object_path,
getattr(cdist_object, 'code_%s_path' % which)) getattr(cdist_object, 'code_%s_path' % which))
return which_exec.run_script(script, env=env) stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which)
stdout_path = os.path.join(cdist_object.stdout_path, 'code-' + which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
return which_exec.run_script(script, stdout=stdout, stderr=stderr)
def run_code_local(self, cdist_object): def run_code_local(self, cdist_object):
"""Run the code-local script for the given cdist object.""" """Run the code-local script for the given cdist object."""

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
@ -153,10 +153,16 @@ class Manifest(object):
message_prefix = "initialmanifest" message_prefix = "initialmanifest"
self.log.verbose("Running initial manifest " + initial_manifest) self.log.verbose("Running initial manifest " + initial_manifest)
self.local.run_script(initial_manifest, which = "init"
stderr_path = os.path.join(self.local.stderr_base_path, which)
stdout_path = os.path.join(self.local.stdout_base_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
self.local.run_script(
initial_manifest,
env=self.env_initial_manifest(initial_manifest), env=self.env_initial_manifest(initial_manifest),
message_prefix=message_prefix, message_prefix=message_prefix,
save_output=False) stdout=stdout, stderr=stderr)
def env_type_manifest(self, cdist_object): def env_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, type_manifest = os.path.join(self.local.type_path,
@ -178,10 +184,16 @@ class Manifest(object):
type_manifest = os.path.join(self.local.type_path, type_manifest = os.path.join(self.local.type_path,
cdist_object.cdist_type.manifest_path) cdist_object.cdist_type.manifest_path)
message_prefix = cdist_object.name message_prefix = cdist_object.name
which = 'manifest'
if os.path.isfile(type_manifest): if os.path.isfile(type_manifest):
self.log.verbose("Running type manifest %s for object %s", self.log.verbose("Running type manifest %s for object %s",
type_manifest, cdist_object.name) type_manifest, cdist_object.name)
self.local.run_script(type_manifest, stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
self.local.run_script(
type_manifest,
env=self.env_type_manifest(cdist_object), env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix, message_prefix=message_prefix,
save_output=False) stdout=stdout, stderr=stderr)

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
@ -120,9 +120,11 @@ class Local(object):
"explorer") "explorer")
self.object_path = os.path.join(self.base_path, "object") self.object_path = os.path.join(self.base_path, "object")
self.messages_path = os.path.join(self.base_path, "messages") self.messages_path = os.path.join(self.base_path, "messages")
self.files_path = os.path.join(self.conf_path, "files") self.stdout_base_path = os.path.join(self.base_path, "stdout")
self.stderr_base_path = os.path.join(self.base_path, "stderr")
# Depending on conf_path # Depending on conf_path
self.files_path = os.path.join(self.conf_path, "files")
self.global_explorer_path = os.path.join(self.conf_path, "explorer") self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest") self.manifest_path = os.path.join(self.conf_path, "manifest")
self.initial_manifest = (self.custom_initial_manifest or self.initial_manifest = (self.custom_initial_manifest or
@ -165,6 +167,8 @@ class Local(object):
self.mkdir(self.object_path) self.mkdir(self.object_path)
self.mkdir(self.bin_path) self.mkdir(self.bin_path)
self.mkdir(self.cache_path) self.mkdir(self.cache_path)
self.mkdir(self.stdout_base_path)
self.mkdir(self.stderr_base_path)
def create_files_dirs(self): def create_files_dirs(self):
self._init_directories() self._init_directories()
@ -199,8 +203,24 @@ class Local(object):
self.log.trace("Local mkdir: %s", path) self.log.trace("Local mkdir: %s", path)
os.makedirs(path, exist_ok=True) os.makedirs(path, exist_ok=True)
def _get_std_fd(self, which):
if which == 'stdout':
base = self.stdout_base_path
else:
base = self.stderr_base_path
path = os.path.join(base, 'remote')
stdfd = open(path, 'ba+')
return stdfd
def _log_std_fd(self, stdfd, which, quiet, save_output):
if not quiet and save_output and stdfd is not None:
stdfd.seek(0, 0)
self.log.trace("Local {}:\n{}\n".format(
which, stdfd.read().decode()))
def run(self, command, env=None, return_output=False, message_prefix=None, def run(self, command, env=None, return_output=False, message_prefix=None,
save_output=True, quiet_mode=False): stdout=None, stderr=None, save_output=True, quiet_mode=False):
"""Run the given command with the given environment. """Run the given command with the given environment.
Return the output as a string. Return the output as a string.
@ -208,6 +228,17 @@ class Local(object):
assert isinstance(command, (list, tuple)), ( assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command) "list or tuple argument expected, got: %s" % command)
quiet = self.quiet_mode or quiet_mode
close_stdout = False
close_stderr = False
if not quiet and save_output and not return_output and stdout is None:
stdout = self._get_std_fd('stdout')
close_stdout = True
if not quiet and save_output and stderr is None:
stderr = self._get_std_fd('stderr')
close_stderr = True
if env is None: if env is None:
env = os.environ.copy() env = os.environ.copy()
# Export __target_host, __target_hostname, __target_fqdn # Export __target_host, __target_hostname, __target_fqdn
@ -225,29 +256,20 @@ class Local(object):
self.log.trace("Local run: %s", command) self.log.trace("Local run: %s", command)
try: try:
if self.quiet_mode or quiet_mode: if quiet:
stderr = subprocess.DEVNULL stderr = subprocess.DEVNULL
else:
stderr = None
if save_output:
output, errout = exec_util.call_get_output(
command, env=env, stderr=stderr)
self.log.trace("Command: {}; local stdout: {}".format(
command, output))
# Currently, stderr is not captured.
# self.log.trace("Local stderr: {}".format(errout))
if return_output: if return_output:
output = subprocess.check_output(
command, env=env, stderr=stderr)
self._log_std_fd(stderr, 'stderr', quiet, save_output)
return output.decode() return output.decode()
else: else:
# In some cases no output is saved. if quiet:
# This is used for shell command, stdout and stderr
# must not be catched.
if self.quiet_mode or quiet_mode:
stdout = subprocess.DEVNULL stdout = subprocess.DEVNULL
else:
stdout = None
subprocess.check_call(command, env=env, stderr=stderr, subprocess.check_call(command, env=env, stderr=stderr,
stdout=stdout) stdout=stdout)
self._log_std_fd(stderr, 'stderr', quiet, save_output)
self._log_std_fd(stdout, 'stdout', quiet, save_output)
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command) exec_util.handle_called_process_error(e, command)
except OSError as error: except OSError as error:
@ -255,9 +277,13 @@ class Local(object):
finally: finally:
if message_prefix: if message_prefix:
message.merge_messages() message.merge_messages()
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()
def run_script(self, script, env=None, return_output=False, def run_script(self, script, env=None, return_output=False,
message_prefix=None, save_output=True): message_prefix=None, stdout=None, stderr=None):
"""Run the given script with the given environment. """Run the given script with the given environment.
Return the output as a string. Return the output as a string.
@ -271,8 +297,9 @@ class Local(object):
script, " ".join(command)) script, " ".join(command))
command.append(script) command.append(script)
return self.run(command=command, env=env, return_output=return_output, return self.run(command, env=env, return_output=return_output,
message_prefix=message_prefix, save_output=save_output) message_prefix=message_prefix, stdout=stdout,
stderr=stderr)
def _cache_subpath_repl(self, matchobj): def _cache_subpath_repl(self, matchobj):
if matchobj.group(2) == '%P': if matchobj.group(2) == '%P':

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
@ -63,7 +63,9 @@ class Remote(object):
base_path=None, base_path=None,
quiet_mode=None, quiet_mode=None,
archiving_mode=None, archiving_mode=None,
configuration=None): configuration=None,
stdout_base_path=None,
stderr_base_path=None):
self.target_host = target_host self.target_host = target_host
self._exec = remote_exec self._exec = remote_exec
self._copy = remote_copy self._copy = remote_copy
@ -79,6 +81,9 @@ class Remote(object):
else: else:
self.configuration = {} self.configuration = {}
self.stdout_base_path = stdout_base_path
self.stderr_base_path = stderr_base_path
self.conf_path = os.path.join(self.base_path, "conf") self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object") self.object_path = os.path.join(self.base_path, "object")
@ -105,7 +110,7 @@ class Remote(object):
self._open_logger() self._open_logger()
def _init_env(self): def _init_env(self):
"""Setup environment for scripts - HERE????""" """Setup environment for scripts."""
# FIXME: better do so in exec functions that require it! # FIXME: better do so in exec functions that require it!
os.environ['__remote_copy'] = self._copy os.environ['__remote_copy'] = self._copy
os.environ['__remote_exec'] = self._exec os.environ['__remote_exec'] = self._exec
@ -237,7 +242,8 @@ class Remote(object):
self.log.trace(("Multiprocessing for parallel transfer " self.log.trace(("Multiprocessing for parallel transfer "
"finished")) "finished"))
def run_script(self, script, env=None, return_output=False): def run_script(self, script, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given script with the given environment on the remote side. """Run the given script with the given environment on the remote side.
Return the output as a string. Return the output as a string.
@ -249,9 +255,11 @@ class Remote(object):
] ]
command.append(script) command.append(script)
return self.run(command, env, return_output) return self.run(command, env=env, return_output=return_output,
stdout=stdout, stderr=stderr)
def run(self, command, env=None, return_output=False): def run(self, command, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given command with the given environment on the remote side. """Run the given command with the given environment on the remote side.
Return the output as a string. Return the output as a string.
@ -284,9 +292,27 @@ class Remote(object):
cmd.append(string_cmd) cmd.append(string_cmd)
else: else:
cmd.extend(command) cmd.extend(command)
return self._run_command(cmd, env=env, return_output=return_output) return self._run_command(cmd, env=env, return_output=return_output,
stdout=stdout, stderr=stderr)
def _run_command(self, command, env=None, return_output=False): def _get_std_fd(self, which):
if which == 'stdout':
base = self.stdout_base_path
else:
base = self.stderr_base_path
path = os.path.join(base, 'remote')
stdfd = open(path, 'ba+')
return stdfd
def _log_std_fd(self, stdfd, which):
if stdfd is not None and stdfd != subprocess.DEVNULL:
stdfd.seek(0, 0)
self.log.trace("Remote {}: {}".format(
which, stdfd.read().decode()))
def _run_command(self, command, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given command with the given environment. """Run the given command with the given environment.
Return the output as a string. Return the output as a string.
@ -294,6 +320,18 @@ class Remote(object):
assert isinstance(command, (list, tuple)), ( assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command) "list or tuple argument expected, got: %s" % command)
if return_output and stdout is not subprocess.PIPE:
self.log.debug("return_output is True, ignoring stdout")
close_stdout = False
close_stderr = False
if not return_output and stdout is None:
stdout = self._get_std_fd('stdout')
close_stdout = True
if stderr is None:
stderr = self._get_std_fd('stderr')
close_stderr = True
# export target_host, target_hostname, target_fqdn # export target_host, target_hostname, target_fqdn
# for use in __remote_{exec,copy} scripts # for use in __remote_{exec,copy} scripts
os_environ = os.environ.copy() os_environ = os.environ.copy()
@ -305,19 +343,24 @@ class Remote(object):
try: try:
if self.quiet_mode: if self.quiet_mode:
stderr = subprocess.DEVNULL stderr = subprocess.DEVNULL
else:
stderr = None
output, errout = exec_util.call_get_output(
command, env=os_environ, stderr=stderr)
self.log.trace("Command: {}; remote stdout: {}".format(
command, output))
# Currently, stderr is not captured.
# self.log.trace("Remote stderr: {}".format(errout))
if return_output: if return_output:
output = subprocess.check_output(command, env=os_environ,
stderr=stderr)
self._log_std_fd(stderr, 'stderr')
return output.decode() return output.decode()
else:
subprocess.check_call(command, env=os_environ, stdout=stdout,
stderr=stderr)
self._log_std_fd(stderr, 'stderr')
self._log_std_fd(stdout, 'stdout')
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command) exec_util.handle_called_process_error(e, command)
except OSError as error: except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1]) raise cdist.Error(" ".join(command) + ": " + error.args[1])
except UnicodeDecodeError: except UnicodeDecodeError:
raise DecodeError(command) raise DecodeError(command)
finally:
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()

View file

@ -89,7 +89,6 @@ class Shell(object):
self._init_environment() self._init_environment()
log.trace("Starting shell...") log.trace("Starting shell...")
# save_output=False -> do not catch stdout and stderr
self.local.run([self.shell], self.env, save_output=False) self.local.run([self.shell], self.env, save_output=False)
log.trace("Finished shell.") log.trace("Finished shell.")

View file

@ -0,0 +1,137 @@
# -*- coding: utf-8 -*-
#
# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import cdist
from cdist import core
from cdist import test
from cdist.exec import local
from cdist.exec import remote
from cdist.core import code
from cdist.core import manifest
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
conf_dir = op.join(fixtures, 'conf')
class CaptureOutputTestCase(test.CdistTestCase):
def setUp(self):
# logging.root.setLevel(logging.TRACE)
self.temp_dir = self.mkdtemp()
self.local_dir = os.path.join(self.temp_dir, "local")
self.hostdir = cdist.str_hash(self.target_host[0])
self.host_base_path = os.path.join(self.local_dir, self.hostdir)
os.makedirs(self.host_base_path)
self.local = local.Local(
target_host=self.target_host,
target_host_tags=None,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
self.remote_dir = self.mkdtemp()
remote_exec = self.remote_exec
remote_copy = self.remote_copy
self.remote = remote.Remote(
target_host=self.target_host,
remote_exec=remote_exec,
remote_copy=remote_copy,
base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote)
self.manifest = manifest.Manifest(self.target_host, self.local)
self.cdist_type = core.CdistType(self.local.type_path,
'__write_to_stdout_and_stderr')
self.cdist_object = core.CdistObject(self.cdist_type,
self.local.object_path,
self.local.object_marker_name,
'')
self.cdist_object.create()
self.output_dirs = {
'object': {
'stdout': os.path.join(self.cdist_object.absolute_path,
'stdout'),
'stderr': os.path.join(self.cdist_object.absolute_path,
'stderr'),
},
'init': {
'stdout': os.path.join(self.local.base_path, 'stdout'),
'stderr': os.path.join(self.local.base_path, 'stderr'),
},
}
def tearDown(self):
shutil.rmtree(self.local_dir)
shutil.rmtree(self.remote_dir)
shutil.rmtree(self.temp_dir)
def _test_output(self, which, target, streams=('stdout', 'stderr')):
for stream in streams:
_should = '{0}: {1}\n'.format(which, stream)
stream_path = os.path.join(self.output_dirs[target][stream], which)
with open(stream_path, 'r') as fd:
_is = fd.read()
self.assertEqual(_should, _is)
def test_capture_code_output(self):
self.cdist_object.code_local = self.code.run_gencode_local(
self.cdist_object)
self._test_output('gencode-local', 'object', ('stderr',))
self.code.run_code_local(self.cdist_object)
self._test_output('code-local', 'object')
self.cdist_object.code_remote = self.code.run_gencode_remote(
self.cdist_object)
self._test_output('gencode-remote', 'object', ('stderr',))
self.code.transfer_code_remote(self.cdist_object)
self.code.run_code_remote(self.cdist_object)
self._test_output('code-remote', 'object')
def test_capture_manifest_output(self):
self.manifest.run_type_manifest(self.cdist_object)
self._test_output('manifest', 'object')
def test_capture_init_manifest_output(self):
initial_manifest = os.path.join(conf_dir, 'manifest', 'init')
self.manifest.run_initial_manifest(initial_manifest)
self._test_output('init', 'init')
if __name__ == "__main__":
import unittest
unittest.main()

View file

@ -0,0 +1,4 @@
#!/bin/sh
echo "init: stdout"
echo "init: stderr" >&2

View file

@ -0,0 +1,6 @@
#!/bin/sh
echo "gencode-local: stderr" >&2
echo "echo \"code-local: stdout\""
echo "echo \"code-local: stderr\" >&2"

View file

@ -0,0 +1,6 @@
#!/bin/sh
echo "gencode-remote: stderr" >&2
echo "echo \"code-remote: stdout\""
echo "echo \"code-remote: stderr\" >&2"

View file

@ -0,0 +1,4 @@
#!/bin/sh
echo "manifest: stdout"
echo "manifest: stderr" >&2

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2012-2015 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
@ -61,7 +61,9 @@ class CodeTestCase(test.CdistTestCase):
target_host=self.target_host, target_host=self.target_host,
remote_exec=remote_exec, remote_exec=remote_exec,
remote_copy=remote_copy, remote_copy=remote_copy,
base_path=self.remote_dir) base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.remote.create_files_dirs() self.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote) self.code = code.Code(self.target_host, self.local, self.remote)
@ -152,6 +154,7 @@ class CodeTestCase(test.CdistTestCase):
self.code.transfer_code_remote(self.cdist_object) self.code.transfer_code_remote(self.cdist_object)
self.code.run_code_remote(self.cdist_object) self.code.run_code_remote(self.cdist_object)
if __name__ == '__main__': if __name__ == '__main__':
import unittest import unittest
unittest.main() unittest.main()

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2010-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2012-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2014 Daniel Heule (hda at sfs.biz) # 2014 Daniel Heule (hda at sfs.biz)
# #
@ -45,6 +45,19 @@ expected_object_names = sorted([
'__third/moon']) '__third/moon'])
class CdistObjectErrorContext(object):
def __init__(self, original_error):
self.original_error = original_error
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, tb):
if exc_type is not None:
if exc_value.original_error:
raise exc_value.original_error
class ConfigRunTestCase(test.CdistTestCase): class ConfigRunTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
@ -87,7 +100,9 @@ class ConfigRunTestCase(test.CdistTestCase):
target_host=self.target_host, target_host=self.target_host,
remote_copy=self.remote_copy, remote_copy=self.remote_copy,
remote_exec=self.remote_exec, remote_exec=self.remote_exec,
base_path=self.remote_dir) base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.local.object_path = self.object_base_path self.local.object_path = self.object_base_path
self.local.type_path = type_base_path self.local.type_path = type_base_path
@ -102,6 +117,20 @@ class ConfigRunTestCase(test.CdistTestCase):
os.environ = self.orig_environ os.environ = self.orig_environ
shutil.rmtree(self.temp_dir) shutil.rmtree(self.temp_dir)
def assertRaisesCdistObjectError(self, original_error, callable_obj):
"""
Test if a raised CdistObjectError was caused by the given
original_error.
"""
with self.assertRaises(original_error):
try:
callable_obj()
except cdist.CdistObjectError as e:
if e.original_error:
raise e.original_error
else:
raise
def test_dependency_resolution(self): def test_dependency_resolution(self):
first = self.object_index['__first/man'] first = self.object_index['__first/man']
second = self.object_index['__second/on-the'] second = self.object_index['__second/on-the']
@ -137,29 +166,33 @@ class ConfigRunTestCase(test.CdistTestCase):
first.requirements = [second.name] first.requirements = [second.name]
second.requirements = [first.name] second.requirements = [first.name]
with self.assertRaises(cdist.UnresolvableRequirementsError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.UnresolvableRequirementsError,
self.config.iterate_until_finished)
def test_missing_requirements(self): def test_missing_requirements(self):
"""Throw an error if requiring something non-existing""" """Throw an error if requiring something non-existing"""
first = self.object_index['__first/man'] first = self.object_index['__first/man']
first.requirements = ['__first/not/exist'] first.requirements = ['__first/not/exist']
with self.assertRaises(cdist.UnresolvableRequirementsError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.UnresolvableRequirementsError,
self.config.iterate_until_finished)
def test_requirement_broken_type(self): def test_requirement_broken_type(self):
"""Unknown type should be detected in the resolving process""" """Unknown type should be detected in the resolving process"""
first = self.object_index['__first/man'] first = self.object_index['__first/man']
first.requirements = ['__nosuchtype/not/exist'] first.requirements = ['__nosuchtype/not/exist']
with self.assertRaises(cdist.core.cdist_type.InvalidTypeError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.core.cdist_type.NoSuchTypeError,
self.config.iterate_until_finished)
def test_requirement_singleton_where_no_singleton(self): def test_requirement_singleton_where_no_singleton(self):
"""Missing object id should be detected in the resolving process""" """Missing object id should be detected in the resolving process"""
first = self.object_index['__first/man'] first = self.object_index['__first/man']
first.requirements = ['__first'] first.requirements = ['__first']
with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.core.cdist_object.MissingObjectIdError,
self.config.iterate_until_finished)
def test_dryrun(self): def test_dryrun(self):
"""Test if the dryrun option is working like expected""" """Test if the dryrun option is working like expected"""

View file

@ -40,12 +40,24 @@ class RemoteTestCase(test.CdistTestCase):
) )
# another temp dir for remote base path # another temp dir for remote base path
self.base_path = self.mkdtemp() self.base_path = self.mkdtemp()
self.remote = self.create_remote()
def create_remote(self, *args, **kwargs):
if not args:
args = (self.target_host,)
kwargs.setdefault('base_path', self.base_path)
user = getpass.getuser() user = getpass.getuser()
remote_exec = "ssh -o User=%s -q" % user kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user)
remote_copy = "scp -o User=%s -q" % user kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user)
self.remote = remote.Remote(self.target_host, base_path=self.base_path, if 'stdout_base_path' not in kwargs:
remote_exec=remote_exec, stdout_path = os.path.join(self.temp_dir, 'stdout')
remote_copy=remote_copy) os.makedirs(stdout_path, exist_ok=True)
kwargs['stdout_base_path'] = stdout_path
if 'stderr_base_path' not in kwargs:
stderr_path = os.path.join(self.temp_dir, 'stderr')
os.makedirs(stderr_path, exist_ok=True)
kwargs['stderr_base_path'] = stderr_path
return remote.Remote(*args, **kwargs)
def tearDown(self): def tearDown(self):
shutil.rmtree(self.temp_dir) shutil.rmtree(self.temp_dir)
@ -155,8 +167,8 @@ class RemoteTestCase(test.CdistTestCase):
os.chmod(remote_exec_path, 0o755) os.chmod(remote_exec_path, 0o755)
remote_exec = remote_exec_path remote_exec = remote_exec_path
remote_copy = "echo" remote_copy = "echo"
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
self.assertEqual(r.run('true', return_output=True), self.assertEqual(r.run('true', return_output=True),
"%s\n" % self.target_host[0]) "%s\n" % self.target_host[0])
@ -167,8 +179,8 @@ class RemoteTestCase(test.CdistTestCase):
os.chmod(remote_exec_path, 0o755) os.chmod(remote_exec_path, 0o755)
remote_exec = remote_exec_path remote_exec = remote_exec_path
remote_copy = "echo" remote_copy = "echo"
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
handle, script = self.mkstemp(dir=self.temp_dir) handle, script = self.mkstemp(dir=self.temp_dir)
with os.fdopen(handle, "w") as fd: with os.fdopen(handle, "w") as fd:
fd.writelines(["#!/bin/sh\n", "true"]) fd.writelines(["#!/bin/sh\n", "true"])
@ -189,8 +201,8 @@ class RemoteTestCase(test.CdistTestCase):
os.chmod(remote_exec_path, 0o755) os.chmod(remote_exec_path, 0o755)
remote_exec = remote_exec_path remote_exec = remote_exec_path
remote_copy = "echo" remote_copy = "echo"
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
output = r.run_script(script, return_output=True) output = r.run_script(script, return_output=True)
self.assertEqual(output, "no_env\n") self.assertEqual(output, "no_env\n")
@ -202,8 +214,8 @@ class RemoteTestCase(test.CdistTestCase):
env = { env = {
'__object': 'test_object', '__object': 'test_object',
} }
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
output = r.run_script(script, env=env, return_output=True) output = r.run_script(script, env=env, return_output=True)
self.assertEqual(output, "test_object\n") self.assertEqual(output, "test_object\n")

View file

@ -64,7 +64,9 @@ class ExplorerClassTestCase(test.CdistTestCase):
target_host=self.target_host, target_host=self.target_host,
remote_exec=self.remote_exec, remote_exec=self.remote_exec,
remote_copy=self.remote_copy, remote_copy=self.remote_copy,
base_path=self.remote_base_path) base_path=self.remote_base_path,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.remote.create_files_dirs() self.remote.create_files_dirs()
self.explorer = explorer.Explorer( self.explorer = explorer.Explorer(

View file

@ -109,6 +109,7 @@ class ManifestTestCase(test.CdistTestCase):
cdist_object = core.CdistObject(cdist_type, self.local.object_path, cdist_object = core.CdistObject(cdist_type, self.local.object_path,
self.local.object_marker_name, self.local.object_marker_name,
'whatever') 'whatever')
cdist_object.create()
handle, output_file = self.mkstemp(dir=self.temp_dir) handle, output_file = self.mkstemp(dir=self.temp_dir)
os.close(handle) os.close(handle)
os.environ['__cdist_test_out'] = output_file os.environ['__cdist_test_out'] = output_file