Compare commits

...

12 commits

Author SHA1 Message Date
Darko Poljak
7f60f70380 Fix after rebase. 2018-01-09 09:16:00 +01:00
Darko Poljak
6956e0b56c Fix changelog 2018-01-09 09:16:00 +01:00
Darko Poljak
4bd1bc11b7 changelog++ 2018-01-09 09:16:00 +01:00
Darko Poljak
21886bd31f Change heading. 2018-01-09 09:16:00 +01:00
Darko Poljak
ca047c7956 Better format error output. 2018-01-09 09:16:00 +01:00
Darko Poljak
7ebd69dcac Refactor init manifest error handling. 2018-01-09 09:16:00 +01:00
Darko Poljak
0f1fa4c041 Handle init manifest error nicely. 2018-01-09 09:16:00 +01:00
Darko Poljak
5813a6bcf3 Add missed env param. 2018-01-09 09:16:00 +01:00
Darko Poljak
7a3c6f2024 Add command to stdout/stderr log. 2018-01-09 09:16:00 +01:00
Darko Poljak
8cfa1c37d0 Fix copy-paste bug. Refactor and simplify code. 2018-01-09 09:16:00 +01:00
Darko Poljak
be4668bf2d Little refactoring. 2018-01-09 09:16:00 +01:00
Steven Armstrong
00cd5fa91e Save output streams. 2018-01-09 09:16:00 +01:00
21 changed files with 483 additions and 120 deletions

View file

@ -1,6 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2012-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -42,7 +43,7 @@ BANNER = """
"P' "" "" "P' "" ""
""" """
REMOTE_COPY = "scp -o User=root" REMOTE_COPY = "scp -o User=root -q"
REMOTE_EXEC = "ssh -o User=root" REMOTE_EXEC = "ssh -o User=root"
REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}"
@ -80,18 +81,73 @@ class CdistBetaRequired(cdist.Error):
return err_msg.format(*fmt_args) return err_msg.format(*fmt_args)
class CdistObjectError(Error): class CdistEntityError(Error):
"""Something went wrong with an object""" """Something went wrong while executing cdist entity"""
def __init__(self, entity_name, entity_params, stderr_paths, subject=''):
self.entity_name = entity_name
self.entity_params = entity_params
self.stderr_paths = stderr_paths
if isinstance(subject, Error):
self.original_error = subject
else:
self.original_error = None
self.message = str(subject)
def __init__(self, cdist_object, message): @property
self.name = cdist_object.name def stderr(self):
self.source = " ".join(cdist_object.source) output = []
self.message = message for stderr_name, stderr_path in self.stderr_paths:
if os.path.getsize(stderr_path) > 0:
label_begin = '---- BEGIN ' + stderr_name + ':stderr ----'
label_end = '---- END ' + stderr_name + ':stderr ----'
output.append('\n' + label_begin)
with open(stderr_path, 'r') as fd:
output.append(fd.read())
output.append(label_end)
return '\n'.join(output)
def __str__(self): def __str__(self):
return '%s: %s (defined at %s)' % (self.name, output = []
self.message, output.append(self.message)
self.source) header = "\nError processing " + self.entity_name
under_header = '=' * len(header)
output.append(header)
output.append(under_header)
for param_name, param_value in self.entity_params:
output.append(param_name + ': ' + str(param_value))
output.append(self.stderr + '\n')
return '\n'.join(output)
class CdistObjectError(CdistEntityError):
"""Something went wrong while working on a specific cdist object"""
def __init__(self, cdist_object, subject=''):
params = [
('name', cdist_object.name, ),
('path', cdist_object.absolute_path, ),
('source', " ".join(cdist_object.source), ),
('type', cdist_object.cdist_type.absolute_path, ),
]
stderr_paths = []
for stderr_name in os.listdir(cdist_object.stderr_path):
stderr_path = os.path.join(cdist_object.stderr_path,
stderr_name)
stderr_paths.append((stderr_name, stderr_path, ))
super().__init__("object '{}'".format(cdist_object.name),
params, stderr_paths, subject)
class InitialManifestError(CdistEntityError):
"""Something went wrong while executing initial manifest"""
def __init__(self, initial_manifest, stderr_path, subject=''):
params = [
('path', initial_manifest, ),
]
stderr_paths = []
stderr_paths = [
('init', stderr_path, ),
]
super().__init__('initial manifest', params, stderr_paths, subject)
def file_to_list(filename): def file_to_list(filename):

View file

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2013-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
@ -353,7 +354,9 @@ class Config(object):
base_path=args.remote_out_path, base_path=args.remote_out_path,
quiet_mode=args.quiet, quiet_mode=args.quiet,
archiving_mode=args.use_archiving, archiving_mode=args.use_archiving,
configuration=configuration) configuration=configuration,
stdout_base_path=local.stdout_base_path,
stderr_base_path=local.stderr_base_path)
cleanup_cmds = [] cleanup_cmds = []
if cleanup_cmd: if cleanup_cmd:
@ -400,7 +403,13 @@ class Config(object):
self._init_files_dirs() self._init_files_dirs()
self.explorer.run_global_explorers(self.local.global_explorer_out_path) self.explorer.run_global_explorers(self.local.global_explorer_out_path)
try:
self.manifest.run_initial_manifest(self.local.initial_manifest) self.manifest.run_initial_manifest(self.local.initial_manifest)
except cdist.Error as e:
which = "init"
stderr_path = os.path.join(self.local.stderr_base_path, which)
raise cdist.InitialManifestError(self.local.initial_manifest,
stderr_path, e)
self.iterate_until_finished() self.iterate_until_finished()
self.cleanup() self.cleanup()
self._remove_files_dirs() self._remove_files_dirs()
@ -453,7 +462,9 @@ class Config(object):
objects_changed = False objects_changed = False
for cdist_object in self.object_list(): for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements): try:
if cdist_object.requirements_unfinished(
cdist_object.requirements):
"""We cannot do anything for this poor object""" """We cannot do anything for this poor object"""
continue continue
@ -463,7 +474,8 @@ class Config(object):
self.object_prepare(cdist_object) self.object_prepare(cdist_object)
objects_changed = True objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire): if cdist_object.requirements_unfinished(
cdist_object.autorequire):
"""The previous step created objects we depend on - """The previous step created objects we depend on -
wait for them wait for them
""" """
@ -472,6 +484,8 @@ class Config(object):
if cdist_object.state == core.CdistObject.STATE_PREPARED: if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object) self.object_run(cdist_object)
objects_changed = True objects_changed = True
except cdist.Error as e:
raise cdist.CdistObjectError(cdist_object, e)
return objects_changed return objects_changed

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2014 Daniel Heule (hda at sfs.biz) # 2014 Daniel Heule (hda at sfs.biz)
# #
@ -80,6 +80,8 @@ class CdistObject(object):
self.code_local_path = os.path.join(self.path, "code-local") self.code_local_path = os.path.join(self.path, "code-local")
self.code_remote_path = os.path.join(self.path, "code-remote") self.code_remote_path = os.path.join(self.path, "code-remote")
self.parameter_path = os.path.join(self.path, "parameter") self.parameter_path = os.path.join(self.path, "parameter")
self.stdout_path = os.path.join(self.absolute_path, "stdout")
self.stderr_path = os.path.join(self.absolute_path, "stderr")
@classmethod @classmethod
def list_objects(cls, object_base_path, type_base_path, object_marker): def list_objects(cls, object_base_path, type_base_path, object_marker):
@ -246,10 +248,11 @@ class CdistObject(object):
"""Create this cdist object on the filesystem. """Create this cdist object on the filesystem.
""" """
try: try:
os.makedirs(self.absolute_path, exist_ok=allow_overwrite) for path in (self.absolute_path,
absolute_parameter_path = os.path.join(self.base_path, os.path.join(self.base_path, self.parameter_path),
self.parameter_path) self.stdout_path,
os.makedirs(absolute_parameter_path, exist_ok=allow_overwrite) self.stderr_path):
os.makedirs(path, exist_ok=allow_overwrite)
except EnvironmentError as error: except EnvironmentError as error:
raise cdist.Error(('Error creating directories for cdist object: ' raise cdist.Error(('Error creating directories for cdist object: '
'%s: %s') % (self, error)) '%s: %s') % (self, error))

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# 2014 Daniel Heule (hda at sfs.biz) # 2014 Daniel Heule (hda at sfs.biz)
# #
@ -127,8 +127,13 @@ class Code(object):
'__object_name': cdist_object.name, '__object_name': cdist_object.name,
}) })
message_prefix = cdist_object.name message_prefix = cdist_object.name
return self.local.run_script(script, env=env, return_output=True, stderr_path = os.path.join(cdist_object.stderr_path,
message_prefix=message_prefix) 'gencode-' + which)
with open(stderr_path, 'ba+') as stderr:
return self.local.run_script(script, env=env,
return_output=True,
message_prefix=message_prefix,
stderr=stderr)
def run_gencode_local(self, cdist_object): def run_gencode_local(self, cdist_object):
"""Run the gencode-local script for the given cdist object.""" """Run the gencode-local script for the given cdist object."""
@ -152,7 +157,12 @@ class Code(object):
which_exec = getattr(self, which) which_exec = getattr(self, which)
script = os.path.join(which_exec.object_path, script = os.path.join(which_exec.object_path,
getattr(cdist_object, 'code_%s_path' % which)) getattr(cdist_object, 'code_%s_path' % which))
return which_exec.run_script(script, env=env) stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which)
stdout_path = os.path.join(cdist_object.stdout_path, 'code-' + which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
return which_exec.run_script(script, env=env, stdout=stdout,
stderr=stderr)
def run_code_local(self, cdist_object): def run_code_local(self, cdist_object):
"""Run the code-local script for the given cdist object.""" """Run the code-local script for the given cdist object."""

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
@ -153,10 +153,16 @@ class Manifest(object):
message_prefix = "initialmanifest" message_prefix = "initialmanifest"
self.log.verbose("Running initial manifest " + initial_manifest) self.log.verbose("Running initial manifest " + initial_manifest)
self.local.run_script(initial_manifest, which = "init"
stderr_path = os.path.join(self.local.stderr_base_path, which)
stdout_path = os.path.join(self.local.stdout_base_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
self.local.run_script(
initial_manifest,
env=self.env_initial_manifest(initial_manifest), env=self.env_initial_manifest(initial_manifest),
message_prefix=message_prefix, message_prefix=message_prefix,
save_output=False) stdout=stdout, stderr=stderr)
def env_type_manifest(self, cdist_object): def env_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, type_manifest = os.path.join(self.local.type_path,
@ -178,10 +184,16 @@ class Manifest(object):
type_manifest = os.path.join(self.local.type_path, type_manifest = os.path.join(self.local.type_path,
cdist_object.cdist_type.manifest_path) cdist_object.cdist_type.manifest_path)
message_prefix = cdist_object.name message_prefix = cdist_object.name
which = 'manifest'
if os.path.isfile(type_manifest): if os.path.isfile(type_manifest):
self.log.verbose("Running type manifest %s for object %s", self.log.verbose("Running type manifest %s for object %s",
type_manifest, cdist_object.name) type_manifest, cdist_object.name)
self.local.run_script(type_manifest, stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
self.local.run_script(
type_manifest,
env=self.env_type_manifest(cdist_object), env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix, message_prefix=message_prefix,
save_output=False) stdout=stdout, stderr=stderr)

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
@ -34,7 +34,7 @@ import datetime
import cdist import cdist
import cdist.message import cdist.message
from cdist import core from cdist import core
import cdist.exec.util as exec_util import cdist.exec.util as util
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
@ -120,9 +120,11 @@ class Local(object):
"explorer") "explorer")
self.object_path = os.path.join(self.base_path, "object") self.object_path = os.path.join(self.base_path, "object")
self.messages_path = os.path.join(self.base_path, "messages") self.messages_path = os.path.join(self.base_path, "messages")
self.files_path = os.path.join(self.conf_path, "files") self.stdout_base_path = os.path.join(self.base_path, "stdout")
self.stderr_base_path = os.path.join(self.base_path, "stderr")
# Depending on conf_path # Depending on conf_path
self.files_path = os.path.join(self.conf_path, "files")
self.global_explorer_path = os.path.join(self.conf_path, "explorer") self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest") self.manifest_path = os.path.join(self.conf_path, "manifest")
self.initial_manifest = (self.custom_initial_manifest or self.initial_manifest = (self.custom_initial_manifest or
@ -165,6 +167,8 @@ class Local(object):
self.mkdir(self.object_path) self.mkdir(self.object_path)
self.mkdir(self.bin_path) self.mkdir(self.bin_path)
self.mkdir(self.cache_path) self.mkdir(self.cache_path)
self.mkdir(self.stdout_base_path)
self.mkdir(self.stderr_base_path)
def create_files_dirs(self): def create_files_dirs(self):
self._init_directories() self._init_directories()
@ -200,7 +204,7 @@ class Local(object):
os.makedirs(path, exist_ok=True) os.makedirs(path, exist_ok=True)
def run(self, command, env=None, return_output=False, message_prefix=None, def run(self, command, env=None, return_output=False, message_prefix=None,
save_output=True, quiet_mode=False): stdout=None, stderr=None, save_output=True, quiet_mode=False):
"""Run the given command with the given environment. """Run the given command with the given environment.
Return the output as a string. Return the output as a string.
@ -208,6 +212,22 @@ class Local(object):
assert isinstance(command, (list, tuple)), ( assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command) "list or tuple argument expected, got: %s" % command)
quiet = self.quiet_mode or quiet_mode
do_save_output = save_output and not quiet
close_stdout = False
close_stderr = False
if quiet:
stderr = subprocess.DEVNULL
stdout = subprocess.DEVNULL
elif do_save_output:
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'local')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'local')
close_stderr = True
if env is None: if env is None:
env = os.environ.copy() env = os.environ.copy()
# Export __target_host, __target_hostname, __target_fqdn # Export __target_host, __target_hostname, __target_fqdn
@ -225,39 +245,33 @@ class Local(object):
self.log.trace("Local run: %s", command) self.log.trace("Local run: %s", command)
try: try:
if self.quiet_mode or quiet_mode:
stderr = subprocess.DEVNULL
else:
stderr = None
if save_output:
output, errout = exec_util.call_get_output(
command, env=env, stderr=stderr)
self.log.trace("Command: {}; local stdout: {}".format(
command, output))
# Currently, stderr is not captured.
# self.log.trace("Local stderr: {}".format(errout))
if return_output: if return_output:
return output.decode() output = subprocess.check_output(
command, env=env, stderr=stderr).decode()
else: else:
# In some cases no output is saved.
# This is used for shell command, stdout and stderr
# must not be catched.
if self.quiet_mode or quiet_mode:
stdout = subprocess.DEVNULL
else:
stdout = None
subprocess.check_call(command, env=env, stderr=stderr, subprocess.check_call(command, env=env, stderr=stderr,
stdout=stdout) stdout=stdout)
output = None
if do_save_output:
util.log_std_fd(self.log, command, stderr, 'Local stderr')
util.log_std_fd(self.log, command, stdout, 'Local stdout')
return output
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command) util.handle_called_process_error(e, command)
except OSError as error: except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1]) raise cdist.Error(" ".join(command) + ": " + error.args[1])
finally: finally:
if message_prefix: if message_prefix:
message.merge_messages() message.merge_messages()
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()
def run_script(self, script, env=None, return_output=False, def run_script(self, script, env=None, return_output=False,
message_prefix=None, save_output=True): message_prefix=None, stdout=None, stderr=None):
"""Run the given script with the given environment. """Run the given script with the given environment.
Return the output as a string. Return the output as a string.
@ -271,8 +285,9 @@ class Local(object):
script, " ".join(command)) script, " ".join(command))
command.append(script) command.append(script)
return self.run(command=command, env=env, return_output=return_output, return self.run(command, env=env, return_output=return_output,
message_prefix=message_prefix, save_output=save_output) message_prefix=message_prefix, stdout=stdout,
stderr=stderr)
def _cache_subpath_repl(self, matchobj): def _cache_subpath_repl(self, matchobj):
if matchobj.group(2) == '%P': if matchobj.group(2) == '%P':

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
@ -27,7 +27,7 @@ import logging
import multiprocessing import multiprocessing
import cdist import cdist
import cdist.exec.util as exec_util import cdist.exec.util as util
import cdist.util.ipaddr as ipaddr import cdist.util.ipaddr as ipaddr
from cdist.mputil import mp_pool_run from cdist.mputil import mp_pool_run
@ -63,7 +63,9 @@ class Remote(object):
base_path=None, base_path=None,
quiet_mode=None, quiet_mode=None,
archiving_mode=None, archiving_mode=None,
configuration=None): configuration=None,
stdout_base_path=None,
stderr_base_path=None):
self.target_host = target_host self.target_host = target_host
self._exec = remote_exec self._exec = remote_exec
self._copy = remote_copy self._copy = remote_copy
@ -79,6 +81,9 @@ class Remote(object):
else: else:
self.configuration = {} self.configuration = {}
self.stdout_base_path = stdout_base_path
self.stderr_base_path = stderr_base_path
self.conf_path = os.path.join(self.base_path, "conf") self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object") self.object_path = os.path.join(self.base_path, "object")
@ -105,7 +110,7 @@ class Remote(object):
self._open_logger() self._open_logger()
def _init_env(self): def _init_env(self):
"""Setup environment for scripts - HERE????""" """Setup environment for scripts."""
# FIXME: better do so in exec functions that require it! # FIXME: better do so in exec functions that require it!
os.environ['__remote_copy'] = self._copy os.environ['__remote_copy'] = self._copy
os.environ['__remote_exec'] = self._exec os.environ['__remote_exec'] = self._exec
@ -237,7 +242,8 @@ class Remote(object):
self.log.trace(("Multiprocessing for parallel transfer " self.log.trace(("Multiprocessing for parallel transfer "
"finished")) "finished"))
def run_script(self, script, env=None, return_output=False): def run_script(self, script, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given script with the given environment on the remote side. """Run the given script with the given environment on the remote side.
Return the output as a string. Return the output as a string.
@ -249,9 +255,11 @@ class Remote(object):
] ]
command.append(script) command.append(script)
return self.run(command, env, return_output) return self.run(command, env=env, return_output=return_output,
stdout=stdout, stderr=stderr)
def run(self, command, env=None, return_output=False): def run(self, command, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given command with the given environment on the remote side. """Run the given command with the given environment on the remote side.
Return the output as a string. Return the output as a string.
@ -284,9 +292,11 @@ class Remote(object):
cmd.append(string_cmd) cmd.append(string_cmd)
else: else:
cmd.extend(command) cmd.extend(command)
return self._run_command(cmd, env=env, return_output=return_output) return self._run_command(cmd, env=env, return_output=return_output,
stdout=stdout, stderr=stderr)
def _run_command(self, command, env=None, return_output=False): def _run_command(self, command, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given command with the given environment. """Run the given command with the given environment.
Return the output as a string. Return the output as a string.
@ -294,6 +304,18 @@ class Remote(object):
assert isinstance(command, (list, tuple)), ( assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command) "list or tuple argument expected, got: %s" % command)
if return_output and stdout is not subprocess.PIPE:
self.log.debug("return_output is True, ignoring stdout")
close_stdout = False
close_stderr = False
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'remote')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'remote')
close_stderr = True
# export target_host, target_hostname, target_fqdn # export target_host, target_hostname, target_fqdn
# for use in __remote_{exec,copy} scripts # for use in __remote_{exec,copy} scripts
os_environ = os.environ.copy() os_environ = os.environ.copy()
@ -305,19 +327,26 @@ class Remote(object):
try: try:
if self.quiet_mode: if self.quiet_mode:
stderr = subprocess.DEVNULL stderr = subprocess.DEVNULL
else:
stderr = None
output, errout = exec_util.call_get_output(
command, env=os_environ, stderr=stderr)
self.log.trace("Command: {}; remote stdout: {}".format(
command, output))
# Currently, stderr is not captured.
# self.log.trace("Remote stderr: {}".format(errout))
if return_output: if return_output:
return output.decode() output = subprocess.check_output(command, env=os_environ,
stderr=stderr).decode()
else:
subprocess.check_call(command, env=os_environ, stdout=stdout,
stderr=stderr)
output = None
util.log_std_fd(self.log, command, stderr, 'Remote stderr')
util.log_std_fd(self.log, command, stdout, 'Remote stdout')
return output
except subprocess.CalledProcessError as e: except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command) util.handle_called_process_error(e, command)
except OSError as error: except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1]) raise cdist.Error(" ".join(command) + ": " + error.args[1])
except UnicodeDecodeError: except UnicodeDecodeError:
raise DecodeError(command) raise DecodeError(command)
finally:
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2016 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -20,6 +20,7 @@
# #
import subprocess import subprocess
import os
from tempfile import TemporaryFile from tempfile import TemporaryFile
import cdist import cdist
@ -115,6 +116,7 @@ import cdist
# return (result.stdout, result.stderr) # return (result.stdout, result.stderr)
# Currently not used.
def call_get_output(command, env=None, stderr=None): def call_get_output(command, env=None, stderr=None):
"""Run the given command with the given environment. """Run the given command with the given environment.
Return the tuple of stdout and stderr output as a byte strings. Return the tuple of stdout and stderr output as a byte strings.
@ -145,6 +147,7 @@ def handle_called_process_error(err, command):
" ".join(command), err.returncode, output)) " ".join(command), err.returncode, output))
# Currently not used.
def _call_get_stdout(command, env=None, stderr=None): def _call_get_stdout(command, env=None, stderr=None):
"""Run the given command with the given environment. """Run the given command with the given environment.
Return the stdout output as a byte string, stderr is ignored. Return the stdout output as a byte string, stderr is ignored.
@ -158,3 +161,16 @@ def _call_get_stdout(command, env=None, stderr=None):
output = fout.read() output = fout.read()
return output return output
def get_std_fd(base_path, name):
path = os.path.join(base_path, name)
stdfd = open(path, 'ba+')
return stdfd
def log_std_fd(log, command, stdfd, prefix):
if stdfd is not None and stdfd != subprocess.DEVNULL:
stdfd.seek(0, 0)
log.trace("Command: {}; {}: {}".format(
command, prefix, stdfd.read().decode()))

View file

@ -89,7 +89,6 @@ class Shell(object):
self._init_environment() self._init_environment()
log.trace("Starting shell...") log.trace("Starting shell...")
# save_output=False -> do not catch stdout and stderr
self.local.run([self.shell], self.env, save_output=False) self.local.run([self.shell], self.env, save_output=False)
log.trace("Finished shell.") log.trace("Finished shell.")

View file

@ -0,0 +1,137 @@
# -*- coding: utf-8 -*-
#
# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import cdist
from cdist import core
from cdist import test
from cdist.exec import local
from cdist.exec import remote
from cdist.core import code
from cdist.core import manifest
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
conf_dir = op.join(fixtures, 'conf')
class CaptureOutputTestCase(test.CdistTestCase):
def setUp(self):
# logging.root.setLevel(logging.TRACE)
self.temp_dir = self.mkdtemp()
self.local_dir = os.path.join(self.temp_dir, "local")
self.hostdir = cdist.str_hash(self.target_host[0])
self.host_base_path = os.path.join(self.local_dir, self.hostdir)
os.makedirs(self.host_base_path)
self.local = local.Local(
target_host=self.target_host,
target_host_tags=None,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
self.remote_dir = self.mkdtemp()
remote_exec = self.remote_exec
remote_copy = self.remote_copy
self.remote = remote.Remote(
target_host=self.target_host,
remote_exec=remote_exec,
remote_copy=remote_copy,
base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote)
self.manifest = manifest.Manifest(self.target_host, self.local)
self.cdist_type = core.CdistType(self.local.type_path,
'__write_to_stdout_and_stderr')
self.cdist_object = core.CdistObject(self.cdist_type,
self.local.object_path,
self.local.object_marker_name,
'')
self.cdist_object.create()
self.output_dirs = {
'object': {
'stdout': os.path.join(self.cdist_object.absolute_path,
'stdout'),
'stderr': os.path.join(self.cdist_object.absolute_path,
'stderr'),
},
'init': {
'stdout': os.path.join(self.local.base_path, 'stdout'),
'stderr': os.path.join(self.local.base_path, 'stderr'),
},
}
def tearDown(self):
shutil.rmtree(self.local_dir)
shutil.rmtree(self.remote_dir)
shutil.rmtree(self.temp_dir)
def _test_output(self, which, target, streams=('stdout', 'stderr')):
for stream in streams:
_should = '{0}: {1}\n'.format(which, stream)
stream_path = os.path.join(self.output_dirs[target][stream], which)
with open(stream_path, 'r') as fd:
_is = fd.read()
self.assertEqual(_should, _is)
def test_capture_code_output(self):
self.cdist_object.code_local = self.code.run_gencode_local(
self.cdist_object)
self._test_output('gencode-local', 'object', ('stderr',))
self.code.run_code_local(self.cdist_object)
self._test_output('code-local', 'object')
self.cdist_object.code_remote = self.code.run_gencode_remote(
self.cdist_object)
self._test_output('gencode-remote', 'object', ('stderr',))
self.code.transfer_code_remote(self.cdist_object)
self.code.run_code_remote(self.cdist_object)
self._test_output('code-remote', 'object')
def test_capture_manifest_output(self):
self.manifest.run_type_manifest(self.cdist_object)
self._test_output('manifest', 'object')
def test_capture_init_manifest_output(self):
initial_manifest = os.path.join(conf_dir, 'manifest', 'init')
self.manifest.run_initial_manifest(initial_manifest)
self._test_output('init', 'init')
if __name__ == "__main__":
import unittest
unittest.main()

View file

@ -0,0 +1,4 @@
#!/bin/sh
echo "init: stdout"
echo "init: stderr" >&2

View file

@ -0,0 +1,6 @@
#!/bin/sh
echo "gencode-local: stderr" >&2
echo "echo \"code-local: stdout\""
echo "echo \"code-local: stderr\" >&2"

View file

@ -0,0 +1,6 @@
#!/bin/sh
echo "gencode-remote: stderr" >&2
echo "echo \"code-remote: stdout\""
echo "echo \"code-remote: stderr\" >&2"

View file

@ -0,0 +1,4 @@
#!/bin/sh
echo "manifest: stdout"
echo "manifest: stderr" >&2

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2012-2015 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
@ -61,7 +61,9 @@ class CodeTestCase(test.CdistTestCase):
target_host=self.target_host, target_host=self.target_host,
remote_exec=remote_exec, remote_exec=remote_exec,
remote_copy=remote_copy, remote_copy=remote_copy,
base_path=self.remote_dir) base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.remote.create_files_dirs() self.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote) self.code = code.Code(self.target_host, self.local, self.remote)
@ -152,6 +154,7 @@ class CodeTestCase(test.CdistTestCase):
self.code.transfer_code_remote(self.cdist_object) self.code.transfer_code_remote(self.cdist_object)
self.code.run_code_remote(self.cdist_object) self.code.run_code_remote(self.cdist_object)
if __name__ == '__main__': if __name__ == '__main__':
import unittest import unittest
unittest.main() unittest.main()

View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2010-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2012-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2014 Daniel Heule (hda at sfs.biz) # 2014 Daniel Heule (hda at sfs.biz)
# #
@ -45,6 +45,19 @@ expected_object_names = sorted([
'__third/moon']) '__third/moon'])
class CdistObjectErrorContext(object):
def __init__(self, original_error):
self.original_error = original_error
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, tb):
if exc_type is not None:
if exc_value.original_error:
raise exc_value.original_error
class ConfigRunTestCase(test.CdistTestCase): class ConfigRunTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
@ -87,7 +100,9 @@ class ConfigRunTestCase(test.CdistTestCase):
target_host=self.target_host, target_host=self.target_host,
remote_copy=self.remote_copy, remote_copy=self.remote_copy,
remote_exec=self.remote_exec, remote_exec=self.remote_exec,
base_path=self.remote_dir) base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.local.object_path = self.object_base_path self.local.object_path = self.object_base_path
self.local.type_path = type_base_path self.local.type_path = type_base_path
@ -102,6 +117,20 @@ class ConfigRunTestCase(test.CdistTestCase):
os.environ = self.orig_environ os.environ = self.orig_environ
shutil.rmtree(self.temp_dir) shutil.rmtree(self.temp_dir)
def assertRaisesCdistObjectError(self, original_error, callable_obj):
"""
Test if a raised CdistObjectError was caused by the given
original_error.
"""
with self.assertRaises(original_error):
try:
callable_obj()
except cdist.CdistObjectError as e:
if e.original_error:
raise e.original_error
else:
raise
def test_dependency_resolution(self): def test_dependency_resolution(self):
first = self.object_index['__first/man'] first = self.object_index['__first/man']
second = self.object_index['__second/on-the'] second = self.object_index['__second/on-the']
@ -137,29 +166,33 @@ class ConfigRunTestCase(test.CdistTestCase):
first.requirements = [second.name] first.requirements = [second.name]
second.requirements = [first.name] second.requirements = [first.name]
with self.assertRaises(cdist.UnresolvableRequirementsError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.UnresolvableRequirementsError,
self.config.iterate_until_finished)
def test_missing_requirements(self): def test_missing_requirements(self):
"""Throw an error if requiring something non-existing""" """Throw an error if requiring something non-existing"""
first = self.object_index['__first/man'] first = self.object_index['__first/man']
first.requirements = ['__first/not/exist'] first.requirements = ['__first/not/exist']
with self.assertRaises(cdist.UnresolvableRequirementsError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.UnresolvableRequirementsError,
self.config.iterate_until_finished)
def test_requirement_broken_type(self): def test_requirement_broken_type(self):
"""Unknown type should be detected in the resolving process""" """Unknown type should be detected in the resolving process"""
first = self.object_index['__first/man'] first = self.object_index['__first/man']
first.requirements = ['__nosuchtype/not/exist'] first.requirements = ['__nosuchtype/not/exist']
with self.assertRaises(cdist.core.cdist_type.InvalidTypeError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.core.cdist_type.InvalidTypeError,
self.config.iterate_until_finished)
def test_requirement_singleton_where_no_singleton(self): def test_requirement_singleton_where_no_singleton(self):
"""Missing object id should be detected in the resolving process""" """Missing object id should be detected in the resolving process"""
first = self.object_index['__first/man'] first = self.object_index['__first/man']
first.requirements = ['__first'] first.requirements = ['__first']
with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): self.assertRaisesCdistObjectError(
self.config.iterate_until_finished() cdist.core.cdist_object.MissingObjectIdError,
self.config.iterate_until_finished)
def test_dryrun(self): def test_dryrun(self):
"""Test if the dryrun option is working like expected""" """Test if the dryrun option is working like expected"""

View file

@ -40,12 +40,24 @@ class RemoteTestCase(test.CdistTestCase):
) )
# another temp dir for remote base path # another temp dir for remote base path
self.base_path = self.mkdtemp() self.base_path = self.mkdtemp()
self.remote = self.create_remote()
def create_remote(self, *args, **kwargs):
if not args:
args = (self.target_host,)
kwargs.setdefault('base_path', self.base_path)
user = getpass.getuser() user = getpass.getuser()
remote_exec = "ssh -o User=%s -q" % user kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user)
remote_copy = "scp -o User=%s -q" % user kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user)
self.remote = remote.Remote(self.target_host, base_path=self.base_path, if 'stdout_base_path' not in kwargs:
remote_exec=remote_exec, stdout_path = os.path.join(self.temp_dir, 'stdout')
remote_copy=remote_copy) os.makedirs(stdout_path, exist_ok=True)
kwargs['stdout_base_path'] = stdout_path
if 'stderr_base_path' not in kwargs:
stderr_path = os.path.join(self.temp_dir, 'stderr')
os.makedirs(stderr_path, exist_ok=True)
kwargs['stderr_base_path'] = stderr_path
return remote.Remote(*args, **kwargs)
def tearDown(self): def tearDown(self):
shutil.rmtree(self.temp_dir) shutil.rmtree(self.temp_dir)
@ -155,8 +167,8 @@ class RemoteTestCase(test.CdistTestCase):
os.chmod(remote_exec_path, 0o755) os.chmod(remote_exec_path, 0o755)
remote_exec = remote_exec_path remote_exec = remote_exec_path
remote_copy = "echo" remote_copy = "echo"
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
self.assertEqual(r.run('true', return_output=True), self.assertEqual(r.run('true', return_output=True),
"%s\n" % self.target_host[0]) "%s\n" % self.target_host[0])
@ -167,8 +179,8 @@ class RemoteTestCase(test.CdistTestCase):
os.chmod(remote_exec_path, 0o755) os.chmod(remote_exec_path, 0o755)
remote_exec = remote_exec_path remote_exec = remote_exec_path
remote_copy = "echo" remote_copy = "echo"
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
handle, script = self.mkstemp(dir=self.temp_dir) handle, script = self.mkstemp(dir=self.temp_dir)
with os.fdopen(handle, "w") as fd: with os.fdopen(handle, "w") as fd:
fd.writelines(["#!/bin/sh\n", "true"]) fd.writelines(["#!/bin/sh\n", "true"])
@ -189,8 +201,8 @@ class RemoteTestCase(test.CdistTestCase):
os.chmod(remote_exec_path, 0o755) os.chmod(remote_exec_path, 0o755)
remote_exec = remote_exec_path remote_exec = remote_exec_path
remote_copy = "echo" remote_copy = "echo"
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
output = r.run_script(script, return_output=True) output = r.run_script(script, return_output=True)
self.assertEqual(output, "no_env\n") self.assertEqual(output, "no_env\n")
@ -202,8 +214,8 @@ class RemoteTestCase(test.CdistTestCase):
env = { env = {
'__object': 'test_object', '__object': 'test_object',
} }
r = remote.Remote(self.target_host, base_path=self.base_path, r = self.create_remote(remote_exec=remote_exec,
remote_exec=remote_exec, remote_copy=remote_copy) remote_copy=remote_copy)
output = r.run_script(script, env=env, return_output=True) output = r.run_script(script, env=env, return_output=True)
self.assertEqual(output, "test_object\n") self.assertEqual(output, "test_object\n")

View file

@ -64,7 +64,9 @@ class ExplorerClassTestCase(test.CdistTestCase):
target_host=self.target_host, target_host=self.target_host,
remote_exec=self.remote_exec, remote_exec=self.remote_exec,
remote_copy=self.remote_copy, remote_copy=self.remote_copy,
base_path=self.remote_base_path) base_path=self.remote_base_path,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path)
self.remote.create_files_dirs() self.remote.create_files_dirs()
self.explorer = explorer.Explorer( self.explorer = explorer.Explorer(

View file

@ -109,6 +109,7 @@ class ManifestTestCase(test.CdistTestCase):
cdist_object = core.CdistObject(cdist_type, self.local.object_path, cdist_object = core.CdistObject(cdist_type, self.local.object_path,
self.local.object_marker_name, self.local.object_marker_name,
'whatever') 'whatever')
cdist_object.create()
handle, output_file = self.mkstemp(dir=self.temp_dir) handle, output_file = self.mkstemp(dir=self.temp_dir)
os.close(handle) os.close(handle)
os.environ['__cdist_test_out'] = output_file os.environ['__cdist_test_out'] = output_file

View file

@ -11,6 +11,7 @@ next:
* Type __letsencrypt_cert: Add nonparallel; make admin-email required (Kamila Součková) * Type __letsencrypt_cert: Add nonparallel; make admin-email required (Kamila Součková)
* Type __package_pkgng_freebsd: Redirect stdout and stderr to /dev/null instead of closing them (michal-hanu-la) * Type __package_pkgng_freebsd: Redirect stdout and stderr to /dev/null instead of closing them (michal-hanu-la)
* Type __daemontools: Make it more robust and clean up the code (Kamila Součková) * Type __daemontools: Make it more robust and clean up the code (Kamila Součková)
* Core: Save output streams (Steven Armstrong, Darko Poljak)
4.7.3: 2017-11-10 4.7.3: 2017-11-10
* Type __ccollect_source: Add create destination parameter (Dominique Roux) * Type __ccollect_source: Add create destination parameter (Dominique Roux)