marcoduif/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests 1 Projects Releases Wiki Activity

Save output streams.

Browse source 
Implementation is 99% based on Steven's initial implementation.
This commit is contained in:
Darko Poljak 2018-01-09 09:31:40 +01:00
commit 9703e0f08e
21 changed files with 484 additions and 121 deletions
Download patch file Download diff file Expand all files Collapse all files

71
cdist/exec/local.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
#
@ -34,7 +34,7 @@ import datetime
import cdist
import cdist.message
from cdist import core
import cdist.exec.util as exec_util
import cdist.exec.util as util
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
@ -120,9 +120,11 @@ class Local(object):
"explorer")
self.object_path = os.path.join(self.base_path, "object")
self.messages_path = os.path.join(self.base_path, "messages")
self.files_path = os.path.join(self.conf_path, "files")
self.stdout_base_path = os.path.join(self.base_path, "stdout")
self.stderr_base_path = os.path.join(self.base_path, "stderr")
# Depending on conf_path
self.files_path = os.path.join(self.conf_path, "files")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.initial_manifest = (self.custom_initial_manifest or
@ -165,6 +167,8 @@ class Local(object):
self.mkdir(self.object_path)
self.mkdir(self.bin_path)
self.mkdir(self.cache_path)
self.mkdir(self.stdout_base_path)
self.mkdir(self.stderr_base_path)
def create_files_dirs(self):
self._init_directories()
@ -200,7 +204,7 @@ class Local(object):
os.makedirs(path, exist_ok=True)
def run(self, command, env=None, return_output=False, message_prefix=None,
save_output=True, quiet_mode=False):
stdout=None, stderr=None, save_output=True, quiet_mode=False):
"""Run the given command with the given environment.
Return the output as a string.
@ -208,6 +212,22 @@ class Local(object):
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command)
quiet = self.quiet_mode or quiet_mode
do_save_output = save_output and not quiet
close_stdout = False
close_stderr = False
if quiet:
stderr = subprocess.DEVNULL
stdout = subprocess.DEVNULL
elif do_save_output:
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'local')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'local')
close_stderr = True
if env is None:
env = os.environ.copy()
# Export __target_host, __target_hostname, __target_fqdn
@ -225,39 +245,33 @@ class Local(object):
self.log.trace("Local run: %s", command)
try:
if self.quiet_mode or quiet_mode:
stderr = subprocess.DEVNULL
if return_output:
output = subprocess.check_output(
command, env=env, stderr=stderr).decode()
else:
stderr = None
if save_output:
output, errout = exec_util.call_get_output(
command, env=env, stderr=stderr)
self.log.trace("Command: {}; local stdout: {}".format(
command, output))
# Currently, stderr is not captured.
# self.log.trace("Local stderr: {}".format(errout))
if return_output:
return output.decode()
else:
# In some cases no output is saved.
# This is used for shell command, stdout and stderr
# must not be catched.
if self.quiet_mode or quiet_mode:
stdout = subprocess.DEVNULL
else:
stdout = None
subprocess.check_call(command, env=env, stderr=stderr,
stdout=stdout)
output = None
if do_save_output:
util.log_std_fd(self.log, command, stderr, 'Local stderr')
util.log_std_fd(self.log, command, stdout, 'Local stdout')
return output
except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command)
util.handle_called_process_error(e, command)
except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])
finally:
if message_prefix:
message.merge_messages()
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()
def run_script(self, script, env=None, return_output=False,
message_prefix=None, save_output=True):
message_prefix=None, stdout=None, stderr=None):
"""Run the given script with the given environment.
Return the output as a string.
@ -271,8 +285,9 @@ class Local(object):
script, " ".join(command))
command.append(script)
return self.run(command=command, env=env, return_output=return_output,
message_prefix=message_prefix, save_output=save_output)
return self.run(command, env=env, return_output=return_output,
message_prefix=message_prefix, stdout=stdout,
stderr=stderr)
def _cache_subpath_repl(self, matchobj):
if matchobj.group(2) == '%P':

67
cdist/exec/remote.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
@ -27,7 +27,7 @@ import logging
import multiprocessing
import cdist
import cdist.exec.util as exec_util
import cdist.exec.util as util
import cdist.util.ipaddr as ipaddr
from cdist.mputil import mp_pool_run
@ -63,7 +63,9 @@ class Remote(object):
base_path=None,
quiet_mode=None,
archiving_mode=None,
configuration=None):
configuration=None,
stdout_base_path=None,
stderr_base_path=None):
self.target_host = target_host
self._exec = remote_exec
self._copy = remote_copy
@ -79,6 +81,9 @@ class Remote(object):
else:
self.configuration = {}
self.stdout_base_path = stdout_base_path
self.stderr_base_path = stderr_base_path
self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object")
@ -105,7 +110,7 @@ class Remote(object):
self._open_logger()
def _init_env(self):
"""Setup environment for scripts - HERE????"""
"""Setup environment for scripts."""
# FIXME: better do so in exec functions that require it!
os.environ['__remote_copy'] = self._copy
os.environ['__remote_exec'] = self._exec
@ -237,7 +242,8 @@ class Remote(object):
self.log.trace(("Multiprocessing for parallel transfer "
"finished"))
def run_script(self, script, env=None, return_output=False):
def run_script(self, script, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given script with the given environment on the remote side.
Return the output as a string.
@ -249,9 +255,11 @@ class Remote(object):
]
command.append(script)
return self.run(command, env, return_output)
return self.run(command, env=env, return_output=return_output,
stdout=stdout, stderr=stderr)
def run(self, command, env=None, return_output=False):
def run(self, command, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given command with the given environment on the remote side.
Return the output as a string.
@ -284,9 +292,11 @@ class Remote(object):
cmd.append(string_cmd)
else:
cmd.extend(command)
return self._run_command(cmd, env=env, return_output=return_output)
return self._run_command(cmd, env=env, return_output=return_output,
stdout=stdout, stderr=stderr)
def _run_command(self, command, env=None, return_output=False):
def _run_command(self, command, env=None, return_output=False, stdout=None,
stderr=None):
"""Run the given command with the given environment.
Return the output as a string.
@ -294,6 +304,18 @@ class Remote(object):
assert isinstance(command, (list, tuple)), (
"list or tuple argument expected, got: %s" % command)
if return_output and stdout is not subprocess.PIPE:
self.log.debug("return_output is True, ignoring stdout")
close_stdout = False
close_stderr = False
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'remote')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'remote')
close_stderr = True
# export target_host, target_hostname, target_fqdn
# for use in __remote_{exec,copy} scripts
os_environ = os.environ.copy()
@ -305,19 +327,26 @@ class Remote(object):
try:
if self.quiet_mode:
stderr = subprocess.DEVNULL
else:
stderr = None
output, errout = exec_util.call_get_output(
command, env=os_environ, stderr=stderr)
self.log.trace("Command: {}; remote stdout: {}".format(
command, output))
# Currently, stderr is not captured.
# self.log.trace("Remote stderr: {}".format(errout))
if return_output:
return output.decode()
output = subprocess.check_output(command, env=os_environ,
stderr=stderr).decode()
else:
subprocess.check_call(command, env=os_environ, stdout=stdout,
stderr=stderr)
output = None
util.log_std_fd(self.log, command, stderr, 'Remote stderr')
util.log_std_fd(self.log, command, stdout, 'Remote stdout')
return output
except subprocess.CalledProcessError as e:
exec_util.handle_called_process_error(e, command)
util.handle_called_process_error(e, command)
except OSError as error:
raise cdist.Error(" ".join(command) + ": " + error.args[1])
except UnicodeDecodeError:
raise DecodeError(command)
finally:
if close_stdout:
stdout.close()
if close_stderr:
stderr.close()

18
cdist/exec/util.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
@ -20,6 +20,7 @@
#
import subprocess
import os
from tempfile import TemporaryFile
import cdist
@ -115,6 +116,7 @@ import cdist
# return (result.stdout, result.stderr)
# Currently not used.
def call_get_output(command, env=None, stderr=None):
"""Run the given command with the given environment.
Return the tuple of stdout and stderr output as a byte strings.
@ -145,6 +147,7 @@ def handle_called_process_error(err, command):
" ".join(command), err.returncode, output))
# Currently not used.
def _call_get_stdout(command, env=None, stderr=None):
"""Run the given command with the given environment.
Return the stdout output as a byte string, stderr is ignored.
@ -158,3 +161,16 @@ def _call_get_stdout(command, env=None, stderr=None):
output = fout.read()
return output
def get_std_fd(base_path, name):
path = os.path.join(base_path, name)
stdfd = open(path, 'ba+')
return stdfd
def log_std_fd(log, command, stdfd, prefix):
if stdfd is not None and stdfd != subprocess.DEVNULL:
stdfd.seek(0, 0)
log.trace("Command: {}; {}: {}".format(
command, prefix, stdfd.read().decode()))