Support disabling saving output streams

This commit is contained in:
Darko Poljak 2018-02-07 18:12:15 +01:00 committed by GitHub
parent 47399bfa9f
commit a993e0f5a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 460 additions and 43 deletions

View file

@ -251,6 +251,10 @@ def get_parsers():
'default.'),
action='store', dest='parallel',
const=multiprocessing.cpu_count())
parser['config_args'].add_argument(
'-S', '--disable-saving-output-streams',
help='Disable saving output streams.',
action='store_false', dest='save_output_streams', default=True)
parser['config_args'].add_argument(
'-s', '--sequential',
help='Operate on multiple hosts sequentially (default).',

View file

@ -345,7 +345,8 @@ class Config(object):
cache_path_pattern=args.cache_path_pattern,
quiet_mode=args.quiet,
configuration=configuration,
exec_path=sys.argv[0])
exec_path=sys.argv[0],
save_output_streams=args.save_output_streams)
remote = cdist.exec.remote.Remote(
target_host=target_host,
@ -356,7 +357,8 @@ class Config(object):
archiving_mode=args.use_archiving,
configuration=configuration,
stdout_base_path=local.stdout_base_path,
stderr_base_path=local.stderr_base_path)
stderr_base_path=local.stderr_base_path,
save_output_streams=args.save_output_streams)
cleanup_cmds = []
if cleanup_cmd:

View file

@ -248,6 +248,7 @@ _ARG_OPTION_MAPPING = {
'parallel': 'parallel',
'verbose': 'verbosity',
'use_archiving': 'archiving',
'save_output_streams': 'save_output_streams',
}
@ -285,6 +286,7 @@ class Configuration(metaclass=Singleton):
'parallel': JobsOption('parallel'),
'verbosity': VerbosityOption(),
'archiving': ArchivingOption(),
'save_output_streams': BooleanOption('save_output_streams'),
},
}
@ -328,7 +330,10 @@ class Configuration(metaclass=Singleton):
config_files=default_config_files, singleton=True):
self.command_line_args = command_line_args
self.args = self._convert_args(command_line_args)
self.env = env
if env is None:
self.env = {}
else:
self.env = env
self.config_files = config_files
self.config = self._get_config()
@ -403,7 +408,8 @@ class Configuration(metaclass=Singleton):
for option in self.ARG_OPTION_MAPPING:
if option in args:
dst_opt = self.ARG_OPTION_MAPPING[option]
if args[option]:
option_object = self.CONFIG_FILE_OPTIONS['GLOBAL'][dst_opt]
if args[option] or isinstance(option_object, BooleanOption):
d[dst_opt] = args[option]
return d

View file

@ -127,13 +127,18 @@ class Code(object):
'__object_name': cdist_object.name,
})
message_prefix = cdist_object.name
stderr_path = os.path.join(cdist_object.stderr_path,
'gencode-' + which)
with open(stderr_path, 'ba+') as stderr:
if self.local.save_output_streams:
stderr_path = os.path.join(cdist_object.stderr_path,
'gencode-' + which)
with open(stderr_path, 'ba+') as stderr:
return self.local.run_script(script, env=env,
return_output=True,
message_prefix=message_prefix,
stderr=stderr)
else:
return self.local.run_script(script, env=env,
return_output=True,
message_prefix=message_prefix,
stderr=stderr)
message_prefix=message_prefix)
def run_gencode_local(self, cdist_object):
"""Run the gencode-local script for the given cdist object."""
@ -157,12 +162,17 @@ class Code(object):
which_exec = getattr(self, which)
script = os.path.join(which_exec.object_path,
getattr(cdist_object, 'code_%s_path' % which))
stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which)
stdout_path = os.path.join(cdist_object.stdout_path, 'code-' + which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
return which_exec.run_script(script, env=env, stdout=stdout,
stderr=stderr)
if which_exec.save_output_streams:
stderr_path = os.path.join(cdist_object.stderr_path,
'code-' + which)
stdout_path = os.path.join(cdist_object.stdout_path,
'code-' + which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
return which_exec.run_script(script, env=env, stdout=stdout,
stderr=stderr)
else:
return which_exec.run_script(script, env=env)
def run_code_local(self, cdist_object):
"""Run the code-local script for the given cdist object."""

View file

@ -154,15 +154,21 @@ class Manifest(object):
message_prefix = "initialmanifest"
self.log.verbose("Running initial manifest " + initial_manifest)
which = "init"
stderr_path = os.path.join(self.local.stderr_base_path, which)
stdout_path = os.path.join(self.local.stdout_base_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
if self.local.save_output_streams:
stderr_path = os.path.join(self.local.stderr_base_path, which)
stdout_path = os.path.join(self.local.stdout_base_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
self.local.run_script(
initial_manifest,
env=self.env_initial_manifest(initial_manifest),
message_prefix=message_prefix,
stdout=stdout, stderr=stderr)
else:
self.local.run_script(
initial_manifest,
env=self.env_initial_manifest(initial_manifest),
message_prefix=message_prefix,
stdout=stdout, stderr=stderr)
message_prefix=message_prefix)
def env_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path,
@ -188,12 +194,18 @@ class Manifest(object):
if os.path.isfile(type_manifest):
self.log.verbose("Running type manifest %s for object %s",
type_manifest, cdist_object.name)
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
if self.local.save_output_streams:
stderr_path = os.path.join(cdist_object.stderr_path, which)
stdout_path = os.path.join(cdist_object.stdout_path, which)
with open(stderr_path, 'ba+') as stderr, \
open(stdout_path, 'ba+') as stdout:
self.local.run_script(
type_manifest,
env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix,
stdout=stdout, stderr=stderr)
else:
self.local.run_script(
type_manifest,
env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix,
stdout=stdout, stderr=stderr)
message_prefix=message_prefix)

View file

@ -56,7 +56,8 @@ class Local(object):
add_conf_dirs=None,
cache_path_pattern=None,
quiet_mode=False,
configuration=None):
configuration=None,
save_output_streams=True):
self.target_host = target_host
if target_host_tags is None:
@ -75,6 +76,7 @@ class Local(object):
self.configuration = configuration
else:
self.configuration = {}
self.save_output_streams = save_output_streams
self._init_log()
self._init_permissions()
@ -213,7 +215,7 @@ class Local(object):
"list or tuple argument expected, got: %s" % command)
quiet = self.quiet_mode or quiet_mode
do_save_output = save_output and not quiet
do_save_output = save_output and not quiet and self.save_output_streams
close_stdout = False
close_stderr = False
@ -256,7 +258,6 @@ class Local(object):
if do_save_output:
util.log_std_fd(self.log, command, stderr, 'Local stderr')
util.log_std_fd(self.log, command, stdout, 'Local stdout')
return output
except subprocess.CalledProcessError as e:
util.handle_called_process_error(e, command)

View file

@ -65,7 +65,8 @@ class Remote(object):
archiving_mode=None,
configuration=None,
stdout_base_path=None,
stderr_base_path=None):
stderr_base_path=None,
save_output_streams=True):
self.target_host = target_host
self._exec = remote_exec
self._copy = remote_copy
@ -80,6 +81,7 @@ class Remote(object):
self.configuration = configuration
else:
self.configuration = {}
self.save_output_streams = save_output_streams
self.stdout_base_path = stdout_base_path
self.stderr_base_path = stderr_base_path
@ -309,12 +311,13 @@ class Remote(object):
close_stdout = False
close_stderr = False
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'remote')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'remote')
close_stderr = True
if self.save_output_streams:
if not return_output and stdout is None:
stdout = util.get_std_fd(self.stdout_base_path, 'remote')
close_stdout = True
if stderr is None:
stderr = util.get_std_fd(self.stderr_base_path, 'remote')
close_stderr = True
# export target_host, target_hostname, target_fqdn
# for use in __remote_{exec,copy} scripts
@ -335,8 +338,9 @@ class Remote(object):
stderr=stderr)
output = None
util.log_std_fd(self.log, command, stderr, 'Remote stderr')
util.log_std_fd(self.log, command, stdout, 'Remote stdout')
if self.save_output_streams:
util.log_std_fd(self.log, command, stderr, 'Remote stderr')
util.log_std_fd(self.log, command, stdout, 'Remote stdout')
return output
except subprocess.CalledProcessError as e:

View file

@ -0,0 +1,140 @@
# -*- coding: utf-8 -*-
#
# 2018 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import os
import shutil
import cdist
from cdist import core
from cdist import test
from cdist.exec import local
from cdist.exec import remote
from cdist.core import code
from cdist.core import manifest
import os.path as op
my_dir = op.abspath(op.dirname(__file__))
fixtures = op.join(my_dir, 'fixtures')
conf_dir = op.join(fixtures, 'conf')
class CaptureOutputDisabledTestCase(test.CdistTestCase):
def setUp(self):
# logging.root.setLevel(logging.TRACE)
save_output_streams = False
self.temp_dir = self.mkdtemp()
self.local_dir = os.path.join(self.temp_dir, "local")
self.hostdir = cdist.str_hash(self.target_host[0])
self.host_base_path = os.path.join(self.local_dir, self.hostdir)
os.makedirs(self.host_base_path)
self.local = local.Local(
target_host=self.target_host,
target_host_tags=None,
base_root_path=self.host_base_path,
host_dir_name=self.hostdir,
exec_path=cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir],
save_output_streams=save_output_streams)
self.local.create_files_dirs()
self.remote_dir = self.mkdtemp()
remote_exec = self.remote_exec
remote_copy = self.remote_copy
self.remote = remote.Remote(
target_host=self.target_host,
remote_exec=remote_exec,
remote_copy=remote_copy,
base_path=self.remote_dir,
stdout_base_path=self.local.stdout_base_path,
stderr_base_path=self.local.stderr_base_path,
save_output_streams=save_output_streams)
self.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote)
self.manifest = manifest.Manifest(self.target_host, self.local)
self.cdist_type = core.CdistType(self.local.type_path,
'__write_to_stdout_and_stderr')
self.cdist_object = core.CdistObject(self.cdist_type,
self.local.object_path,
self.local.object_marker_name,
'')
self.cdist_object.create()
self.output_dirs = {
'object': {
'stdout': os.path.join(self.cdist_object.absolute_path,
'stdout'),
'stderr': os.path.join(self.cdist_object.absolute_path,
'stderr'),
},
'init': {
'stdout': os.path.join(self.local.base_path, 'stdout'),
'stderr': os.path.join(self.local.base_path, 'stderr'),
},
}
def tearDown(self):
shutil.rmtree(self.local_dir)
shutil.rmtree(self.remote_dir)
shutil.rmtree(self.temp_dir)
def _test_output(self, which, target, streams=('stdout', 'stderr')):
for stream in streams:
stream_path = os.path.join(self.output_dirs[target][stream], which)
if os.path.exists(stream_path):
with open(stream_path, 'r') as fd:
_is = fd.read()
self.assertEqual("", _is)
# else ok when not exists
def test_capture_code_output_disabled(self):
self.cdist_object.code_local = self.code.run_gencode_local(
self.cdist_object)
self._test_output('gencode-local', 'object', ('stderr',))
self.code.run_code_local(self.cdist_object)
self._test_output('code-local', 'object')
self.cdist_object.code_remote = self.code.run_gencode_remote(
self.cdist_object)
self._test_output('gencode-remote', 'object', ('stderr',))
self.code.transfer_code_remote(self.cdist_object)
self.code.run_code_remote(self.cdist_object)
self._test_output('code-remote', 'object')
def test_capture_manifest_output_disabled(self):
self.manifest.run_type_manifest(self.cdist_object)
self._test_output('manifest', 'object')
def test_capture_init_manifest_output_disabled(self):
initial_manifest = os.path.join(conf_dir, 'manifest', 'init')
self.manifest.run_initial_manifest(initial_manifest)
self._test_output('init', 'init')
if __name__ == "__main__":
import unittest
unittest.main()

View file

@ -0,0 +1,4 @@
#!/bin/sh
echo "init: stdout"
echo "init: stderr" >&2

View file

@ -0,0 +1,6 @@
#!/bin/sh
echo "gencode-local: stderr" >&2
echo "echo \"code-local: stdout\""
echo "echo \"code-local: stderr\" >&2"

View file

@ -0,0 +1,6 @@
#!/bin/sh
echo "gencode-remote: stderr" >&2
echo "echo \"code-remote: stdout\""
echo "echo \"code-remote: stderr\" >&2"

View file

@ -0,0 +1,4 @@
#!/bin/sh
echo "manifest: stdout"
echo "manifest: stderr" >&2

View file

@ -300,6 +300,7 @@ class ConfigurationTestCase(test.CdistTestCase):
expected = {
'conf_dir': ['/usr/local/cdist1', ],
'verbosity': 3,
'beta': False,
}
args_dict = vars(args)
d = config._read_args_config(args_dict)
@ -1167,6 +1168,118 @@ class ConfigurationTestCase(test.CdistTestCase):
configuration = cc.Configuration(args, env=env,
config_files=())
def test_configuration_disable_saving_output_streams1(self):
config = configparser.ConfigParser()
config['GLOBAL'] = {
'save_output_streams': 'True',
}
global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
with open(global_config_file, 'w') as f:
config.write(f)
expected_config_dict = {
'GLOBAL': {
'save_output_streams': True,
'verbosity': 0,
},
}
config_files = (global_config_file, )
# bypass singleton so we can test further
cc.Configuration.instance = None
args = argparse.Namespace()
args.save_output_streams = True
configuration = cc.Configuration(args, env=None,
config_files=config_files)
self.assertEqual(configuration.config, expected_config_dict)
def test_configuration_disable_saving_output_streams2(self):
config = configparser.ConfigParser()
config['GLOBAL'] = {
'save_output_streams': 'False',
}
global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
with open(global_config_file, 'w') as f:
config.write(f)
expected_config_dict = {
'GLOBAL': {
'save_output_streams': True,
'verbosity': 0,
},
}
config_files = (global_config_file, )
# bypass singleton so we can test further
cc.Configuration.instance = None
args = argparse.Namespace()
args.save_output_streams = True
configuration = cc.Configuration(args, env=None,
config_files=config_files)
self.assertEqual(configuration.config, expected_config_dict)
def test_configuration_disable_saving_output_streams3(self):
config = configparser.ConfigParser()
config['GLOBAL'] = {
'save_output_streams': 'False',
}
global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
with open(global_config_file, 'w') as f:
config.write(f)
expected_config_dict = {
'GLOBAL': {
'save_output_streams': False,
'verbosity': 0,
},
}
config_files = (global_config_file, )
# bypass singleton so we can test further
cc.Configuration.instance = None
args = argparse.Namespace()
args.save_output_streams = False
configuration = cc.Configuration(args, env=None,
config_files=config_files)
self.assertEqual(configuration.config, expected_config_dict)
def test_configuration_disable_saving_output_streams4(self):
config = configparser.ConfigParser()
config['GLOBAL'] = {
'save_output_streams': 'True',
}
global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
with open(global_config_file, 'w') as f:
config.write(f)
expected_config_dict = {
'GLOBAL': {
'save_output_streams': False,
'verbosity': 0,
},
}
config_files = (global_config_file, )
# bypass singleton so we can test further
cc.Configuration.instance = None
args = argparse.Namespace()
args.save_output_streams = False
configuration = cc.Configuration(args, env=None,
config_files=config_files)
self.assertEqual(configuration.config, expected_config_dict)
if __name__ == "__main__":
import unittest

View file

@ -87,10 +87,12 @@ state
this type execution state ('done' when finished)
stderr
directory containing type's gencode-* and code-* stderr stream outputs
directory containing type's manifest, gencode-* and code-* stderr stream
outputs
stdin
this type stdin content
stdout
directory containing type's gencode-* and code-* stdout stream outputs.
directory containing type's manifest, gencode-* and code-* stdout stream
outputs.

View file

@ -88,6 +88,11 @@ The possible keywords and their meanings are as follows:
:strong:`remote_shell`
Shell command at remote host used for remote execution.
:strong:`save_output_streams`
Enable/disable saving output streams (enabled by default).
It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false'
and '1'/'0'.
:strong:`verbosity`
Set verbosity level. Valid values are:
'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.

View file

@ -0,0 +1,88 @@
Saving output streams
=====================
Description
-----------
Since version 4.8.0 cdist, by default, saves output streams to local cache.
Saving output streams is implemented because important information was lost
during a config run, hidden in all other output.
Now all created output is bound to the context where it was produced.
Saving output streams include stdout and stderr of init manifest, remote
commands and for each object stdout and stderr of manifest, gencode-* and code-*.
Output stream files are created only if some output is produced. For more info
on these cache files see `Local cache overview <cdist-cache.html>`_.
Also, in case of an error, cdist can now exit and show all information it has
about the error.
For example:
.. code-block:: sh
$ ./bin/cdist config -v -i ~/.cdist/manifest/init-output-streams $(cat ~/ungleich/data/opennebula-debian9-test )
INFO: 185.203.112.42: Starting configuration run
INFO: 185.203.112.42: Processing __myline/test
ERROR: 185.203.112.42: Command failed: '/bin/sh -e /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-kisrqlpw/code-local'
return code: 1
---- BEGIN stdout ----
---- END stdout ----
Error processing object '__myline/test'
========================================
name: __myline/test
path: /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-kisrqlpw
source: /home/darko/.cdist/manifest/init-output-streams
type: /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/conf/type/__myline
---- BEGIN manifest:stderr ----
myline manifest stderr
---- END manifest:stderr ----
---- BEGIN gencode-remote:stderr ----
test gencode-remote error
---- END gencode-remote:stderr ----
---- BEGIN code-local:stderr ----
error
---- END code-local:stderr ----
ERROR: cdist: Failed to configure the following hosts: 185.203.112.42
Upon successful run execution state is saved to local cache and temporary
directory is removed.
In case of an error temporary directory is not removed and can be further
discovered.
There is also an option :strong:`-S/--disable-saving-output-streams` for
disabling saving output streams. In this case error reporting can look
like this:
.. code-block:: sh
$ ./bin/cdist config -v -S -i ~/.cdist/manifest/init-output-streams $(cat ~/ungleich/data/opennebula-debian9-test )
INFO: 185.203.112.42: Starting configuration run
test stdout output streams
test stderr output streams
myline manifest stdout
myline manifest stderr
test gencode-remote error
INFO: 185.203.112.42: Processing __myline/test
error
ERROR: 185.203.112.42: Command failed: '/bin/sh -e /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-n566pqut/code-local'
return code: 1
---- BEGIN stdout ----
---- END stdout ----
Error processing object '__myline/test'
========================================
name: __myline/test
path: /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-n566pqut
source: /home/darko/.cdist/manifest/init-output-streams
type: /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/conf/type/__myline
ERROR: cdist: Failed to configure the following hosts: 185.203.112.42

View file

@ -31,6 +31,7 @@ Contents:
cdist-best-practice
cdist-stages
cdist-cache
cdist-saving-output-streams
cdist-remote-exec-copy
cdist-hacker
cdist-troubleshooting

View file

@ -21,7 +21,7 @@ SYNOPSIS
[-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
[--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
[-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[-f HOSTFILE] [-p [HOST_MAX]] [-S] [-s] [-t]
[host [host ...]]
cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
@ -29,7 +29,7 @@ SYNOPSIS
[-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
[-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
[--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
[-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
[-f HOSTFILE] [-p [HOST_MAX]] [-S] [-s] [-t]
[host [host ...]]
cdist inventory [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
@ -200,6 +200,10 @@ Install command is currently in beta.
Directory to save cdist output in on the target host.
.. option:: -S, --disable-saving-output-streams
Disable saving output streams.
.. option:: -s, --sequential
Operate on multiple hosts sequentially (default).
@ -561,6 +565,11 @@ The possible keywords and their meanings are as follows:
:strong:`remote_shell`
Shell command at remote host used for remote execution.
:strong:`save_output_streams`
Enable/disable saving output streams (enabled by default).
It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false'
and '1'/'0'.
:strong:`verbosity`
Set verbosity level. Valid values are:
'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.