Support disabling saving output streams
This commit is contained in:
		
					parent
					
						
							
								47399bfa9f
							
						
					
				
			
			
				commit
				
					
						a993e0f5a9
					
				
			
		
					 19 changed files with 460 additions and 43 deletions
				
			
		| 
						 | 
				
			
			@ -251,6 +251,10 @@ def get_parsers():
 | 
			
		|||
                 'default.'),
 | 
			
		||||
           action='store', dest='parallel',
 | 
			
		||||
           const=multiprocessing.cpu_count())
 | 
			
		||||
    parser['config_args'].add_argument(
 | 
			
		||||
           '-S', '--disable-saving-output-streams',
 | 
			
		||||
           help='Disable saving output streams.',
 | 
			
		||||
           action='store_false', dest='save_output_streams', default=True)
 | 
			
		||||
    parser['config_args'].add_argument(
 | 
			
		||||
           '-s', '--sequential',
 | 
			
		||||
           help='Operate on multiple hosts sequentially (default).',
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -345,7 +345,8 @@ class Config(object):
 | 
			
		|||
                cache_path_pattern=args.cache_path_pattern,
 | 
			
		||||
                quiet_mode=args.quiet,
 | 
			
		||||
                configuration=configuration,
 | 
			
		||||
                exec_path=sys.argv[0])
 | 
			
		||||
                exec_path=sys.argv[0],
 | 
			
		||||
                save_output_streams=args.save_output_streams)
 | 
			
		||||
 | 
			
		||||
            remote = cdist.exec.remote.Remote(
 | 
			
		||||
                target_host=target_host,
 | 
			
		||||
| 
						 | 
				
			
			@ -356,7 +357,8 @@ class Config(object):
 | 
			
		|||
                archiving_mode=args.use_archiving,
 | 
			
		||||
                configuration=configuration,
 | 
			
		||||
                stdout_base_path=local.stdout_base_path,
 | 
			
		||||
                stderr_base_path=local.stderr_base_path)
 | 
			
		||||
                stderr_base_path=local.stderr_base_path,
 | 
			
		||||
                save_output_streams=args.save_output_streams)
 | 
			
		||||
 | 
			
		||||
            cleanup_cmds = []
 | 
			
		||||
            if cleanup_cmd:
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -248,6 +248,7 @@ _ARG_OPTION_MAPPING = {
 | 
			
		|||
    'parallel': 'parallel',
 | 
			
		||||
    'verbose': 'verbosity',
 | 
			
		||||
    'use_archiving': 'archiving',
 | 
			
		||||
    'save_output_streams': 'save_output_streams',
 | 
			
		||||
}
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -285,6 +286,7 @@ class Configuration(metaclass=Singleton):
 | 
			
		|||
            'parallel': JobsOption('parallel'),
 | 
			
		||||
            'verbosity': VerbosityOption(),
 | 
			
		||||
            'archiving': ArchivingOption(),
 | 
			
		||||
            'save_output_streams': BooleanOption('save_output_streams'),
 | 
			
		||||
        },
 | 
			
		||||
    }
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -328,7 +330,10 @@ class Configuration(metaclass=Singleton):
 | 
			
		|||
                 config_files=default_config_files, singleton=True):
 | 
			
		||||
        self.command_line_args = command_line_args
 | 
			
		||||
        self.args = self._convert_args(command_line_args)
 | 
			
		||||
        self.env = env
 | 
			
		||||
        if env is None:
 | 
			
		||||
            self.env = {}
 | 
			
		||||
        else:
 | 
			
		||||
            self.env = env
 | 
			
		||||
        self.config_files = config_files
 | 
			
		||||
        self.config = self._get_config()
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			@ -403,7 +408,8 @@ class Configuration(metaclass=Singleton):
 | 
			
		|||
        for option in self.ARG_OPTION_MAPPING:
 | 
			
		||||
            if option in args:
 | 
			
		||||
                dst_opt = self.ARG_OPTION_MAPPING[option]
 | 
			
		||||
                if args[option]:
 | 
			
		||||
                option_object = self.CONFIG_FILE_OPTIONS['GLOBAL'][dst_opt]
 | 
			
		||||
                if args[option] or isinstance(option_object, BooleanOption):
 | 
			
		||||
                    d[dst_opt] = args[option]
 | 
			
		||||
        return d
 | 
			
		||||
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -127,13 +127,18 @@ class Code(object):
 | 
			
		|||
                '__object_name': cdist_object.name,
 | 
			
		||||
            })
 | 
			
		||||
            message_prefix = cdist_object.name
 | 
			
		||||
            stderr_path = os.path.join(cdist_object.stderr_path,
 | 
			
		||||
                                       'gencode-' + which)
 | 
			
		||||
            with open(stderr_path, 'ba+') as stderr:
 | 
			
		||||
            if self.local.save_output_streams:
 | 
			
		||||
                stderr_path = os.path.join(cdist_object.stderr_path,
 | 
			
		||||
                                           'gencode-' + which)
 | 
			
		||||
                with open(stderr_path, 'ba+') as stderr:
 | 
			
		||||
                    return self.local.run_script(script, env=env,
 | 
			
		||||
                                                 return_output=True,
 | 
			
		||||
                                                 message_prefix=message_prefix,
 | 
			
		||||
                                                 stderr=stderr)
 | 
			
		||||
            else:
 | 
			
		||||
                return self.local.run_script(script, env=env,
 | 
			
		||||
                                             return_output=True,
 | 
			
		||||
                                             message_prefix=message_prefix,
 | 
			
		||||
                                             stderr=stderr)
 | 
			
		||||
                                             message_prefix=message_prefix)
 | 
			
		||||
 | 
			
		||||
    def run_gencode_local(self, cdist_object):
 | 
			
		||||
        """Run the gencode-local script for the given cdist object."""
 | 
			
		||||
| 
						 | 
				
			
			@ -157,12 +162,17 @@ class Code(object):
 | 
			
		|||
        which_exec = getattr(self, which)
 | 
			
		||||
        script = os.path.join(which_exec.object_path,
 | 
			
		||||
                              getattr(cdist_object, 'code_%s_path' % which))
 | 
			
		||||
        stderr_path = os.path.join(cdist_object.stderr_path, 'code-' + which)
 | 
			
		||||
        stdout_path = os.path.join(cdist_object.stdout_path, 'code-' + which)
 | 
			
		||||
        with open(stderr_path, 'ba+') as stderr, \
 | 
			
		||||
                open(stdout_path, 'ba+') as stdout:
 | 
			
		||||
            return which_exec.run_script(script, env=env, stdout=stdout,
 | 
			
		||||
                                         stderr=stderr)
 | 
			
		||||
        if which_exec.save_output_streams:
 | 
			
		||||
            stderr_path = os.path.join(cdist_object.stderr_path,
 | 
			
		||||
                                       'code-' + which)
 | 
			
		||||
            stdout_path = os.path.join(cdist_object.stdout_path,
 | 
			
		||||
                                       'code-' + which)
 | 
			
		||||
            with open(stderr_path, 'ba+') as stderr, \
 | 
			
		||||
                    open(stdout_path, 'ba+') as stdout:
 | 
			
		||||
                return which_exec.run_script(script, env=env, stdout=stdout,
 | 
			
		||||
                                             stderr=stderr)
 | 
			
		||||
        else:
 | 
			
		||||
            return which_exec.run_script(script, env=env)
 | 
			
		||||
 | 
			
		||||
    def run_code_local(self, cdist_object):
 | 
			
		||||
        """Run the code-local script for the given cdist object."""
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -154,15 +154,21 @@ class Manifest(object):
 | 
			
		|||
        message_prefix = "initialmanifest"
 | 
			
		||||
        self.log.verbose("Running initial manifest " + initial_manifest)
 | 
			
		||||
        which = "init"
 | 
			
		||||
        stderr_path = os.path.join(self.local.stderr_base_path, which)
 | 
			
		||||
        stdout_path = os.path.join(self.local.stdout_base_path, which)
 | 
			
		||||
        with open(stderr_path, 'ba+') as stderr, \
 | 
			
		||||
                open(stdout_path, 'ba+') as stdout:
 | 
			
		||||
        if self.local.save_output_streams:
 | 
			
		||||
            stderr_path = os.path.join(self.local.stderr_base_path, which)
 | 
			
		||||
            stdout_path = os.path.join(self.local.stdout_base_path, which)
 | 
			
		||||
            with open(stderr_path, 'ba+') as stderr, \
 | 
			
		||||
                    open(stdout_path, 'ba+') as stdout:
 | 
			
		||||
                self.local.run_script(
 | 
			
		||||
                    initial_manifest,
 | 
			
		||||
                    env=self.env_initial_manifest(initial_manifest),
 | 
			
		||||
                    message_prefix=message_prefix,
 | 
			
		||||
                    stdout=stdout, stderr=stderr)
 | 
			
		||||
        else:
 | 
			
		||||
            self.local.run_script(
 | 
			
		||||
                initial_manifest,
 | 
			
		||||
                env=self.env_initial_manifest(initial_manifest),
 | 
			
		||||
                message_prefix=message_prefix,
 | 
			
		||||
                stdout=stdout, stderr=stderr)
 | 
			
		||||
                message_prefix=message_prefix)
 | 
			
		||||
 | 
			
		||||
    def env_type_manifest(self, cdist_object):
 | 
			
		||||
        type_manifest = os.path.join(self.local.type_path,
 | 
			
		||||
| 
						 | 
				
			
			@ -188,12 +194,18 @@ class Manifest(object):
 | 
			
		|||
        if os.path.isfile(type_manifest):
 | 
			
		||||
            self.log.verbose("Running type manifest %s for object %s",
 | 
			
		||||
                             type_manifest, cdist_object.name)
 | 
			
		||||
            stderr_path = os.path.join(cdist_object.stderr_path, which)
 | 
			
		||||
            stdout_path = os.path.join(cdist_object.stdout_path, which)
 | 
			
		||||
            with open(stderr_path, 'ba+') as stderr, \
 | 
			
		||||
                    open(stdout_path, 'ba+') as stdout:
 | 
			
		||||
            if self.local.save_output_streams:
 | 
			
		||||
                stderr_path = os.path.join(cdist_object.stderr_path, which)
 | 
			
		||||
                stdout_path = os.path.join(cdist_object.stdout_path, which)
 | 
			
		||||
                with open(stderr_path, 'ba+') as stderr, \
 | 
			
		||||
                        open(stdout_path, 'ba+') as stdout:
 | 
			
		||||
                    self.local.run_script(
 | 
			
		||||
                        type_manifest,
 | 
			
		||||
                        env=self.env_type_manifest(cdist_object),
 | 
			
		||||
                        message_prefix=message_prefix,
 | 
			
		||||
                        stdout=stdout, stderr=stderr)
 | 
			
		||||
            else:
 | 
			
		||||
                self.local.run_script(
 | 
			
		||||
                    type_manifest,
 | 
			
		||||
                    env=self.env_type_manifest(cdist_object),
 | 
			
		||||
                    message_prefix=message_prefix,
 | 
			
		||||
                    stdout=stdout, stderr=stderr)
 | 
			
		||||
                    message_prefix=message_prefix)
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -56,7 +56,8 @@ class Local(object):
 | 
			
		|||
                 add_conf_dirs=None,
 | 
			
		||||
                 cache_path_pattern=None,
 | 
			
		||||
                 quiet_mode=False,
 | 
			
		||||
                 configuration=None):
 | 
			
		||||
                 configuration=None,
 | 
			
		||||
                 save_output_streams=True):
 | 
			
		||||
 | 
			
		||||
        self.target_host = target_host
 | 
			
		||||
        if target_host_tags is None:
 | 
			
		||||
| 
						 | 
				
			
			@ -75,6 +76,7 @@ class Local(object):
 | 
			
		|||
            self.configuration = configuration
 | 
			
		||||
        else:
 | 
			
		||||
            self.configuration = {}
 | 
			
		||||
        self.save_output_streams = save_output_streams
 | 
			
		||||
 | 
			
		||||
        self._init_log()
 | 
			
		||||
        self._init_permissions()
 | 
			
		||||
| 
						 | 
				
			
			@ -213,7 +215,7 @@ class Local(object):
 | 
			
		|||
                "list or tuple argument expected, got: %s" % command)
 | 
			
		||||
 | 
			
		||||
        quiet = self.quiet_mode or quiet_mode
 | 
			
		||||
        do_save_output = save_output and not quiet
 | 
			
		||||
        do_save_output = save_output and not quiet and self.save_output_streams
 | 
			
		||||
 | 
			
		||||
        close_stdout = False
 | 
			
		||||
        close_stderr = False
 | 
			
		||||
| 
						 | 
				
			
			@ -256,7 +258,6 @@ class Local(object):
 | 
			
		|||
            if do_save_output:
 | 
			
		||||
                util.log_std_fd(self.log, command, stderr, 'Local stderr')
 | 
			
		||||
                util.log_std_fd(self.log, command, stdout, 'Local stdout')
 | 
			
		||||
 | 
			
		||||
            return output
 | 
			
		||||
        except subprocess.CalledProcessError as e:
 | 
			
		||||
            util.handle_called_process_error(e, command)
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -65,7 +65,8 @@ class Remote(object):
 | 
			
		|||
                 archiving_mode=None,
 | 
			
		||||
                 configuration=None,
 | 
			
		||||
                 stdout_base_path=None,
 | 
			
		||||
                 stderr_base_path=None):
 | 
			
		||||
                 stderr_base_path=None,
 | 
			
		||||
                 save_output_streams=True):
 | 
			
		||||
        self.target_host = target_host
 | 
			
		||||
        self._exec = remote_exec
 | 
			
		||||
        self._copy = remote_copy
 | 
			
		||||
| 
						 | 
				
			
			@ -80,6 +81,7 @@ class Remote(object):
 | 
			
		|||
            self.configuration = configuration
 | 
			
		||||
        else:
 | 
			
		||||
            self.configuration = {}
 | 
			
		||||
        self.save_output_streams = save_output_streams
 | 
			
		||||
 | 
			
		||||
        self.stdout_base_path = stdout_base_path
 | 
			
		||||
        self.stderr_base_path = stderr_base_path
 | 
			
		||||
| 
						 | 
				
			
			@ -309,12 +311,13 @@ class Remote(object):
 | 
			
		|||
 | 
			
		||||
        close_stdout = False
 | 
			
		||||
        close_stderr = False
 | 
			
		||||
        if not return_output and stdout is None:
 | 
			
		||||
            stdout = util.get_std_fd(self.stdout_base_path, 'remote')
 | 
			
		||||
            close_stdout = True
 | 
			
		||||
        if stderr is None:
 | 
			
		||||
            stderr = util.get_std_fd(self.stderr_base_path, 'remote')
 | 
			
		||||
            close_stderr = True
 | 
			
		||||
        if self.save_output_streams:
 | 
			
		||||
            if not return_output and stdout is None:
 | 
			
		||||
                stdout = util.get_std_fd(self.stdout_base_path, 'remote')
 | 
			
		||||
                close_stdout = True
 | 
			
		||||
            if stderr is None:
 | 
			
		||||
                stderr = util.get_std_fd(self.stderr_base_path, 'remote')
 | 
			
		||||
                close_stderr = True
 | 
			
		||||
 | 
			
		||||
        # export target_host, target_hostname, target_fqdn
 | 
			
		||||
        # for use in __remote_{exec,copy} scripts
 | 
			
		||||
| 
						 | 
				
			
			@ -335,8 +338,9 @@ class Remote(object):
 | 
			
		|||
                                      stderr=stderr)
 | 
			
		||||
                output = None
 | 
			
		||||
 | 
			
		||||
            util.log_std_fd(self.log, command, stderr, 'Remote stderr')
 | 
			
		||||
            util.log_std_fd(self.log, command, stdout, 'Remote stdout')
 | 
			
		||||
            if self.save_output_streams:
 | 
			
		||||
                util.log_std_fd(self.log, command, stderr, 'Remote stderr')
 | 
			
		||||
                util.log_std_fd(self.log, command, stdout, 'Remote stdout')
 | 
			
		||||
 | 
			
		||||
            return output
 | 
			
		||||
        except subprocess.CalledProcessError as e:
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										140
									
								
								cdist/test/capture_output_disabled/__init__.py
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										140
									
								
								cdist/test/capture_output_disabled/__init__.py
									
										
									
									
									
										Normal file
									
								
							| 
						 | 
				
			
			@ -0,0 +1,140 @@
 | 
			
		|||
# -*- coding: utf-8 -*-
 | 
			
		||||
#
 | 
			
		||||
# 2018 Darko Poljak (darko.poljak at gmail.com)
 | 
			
		||||
#
 | 
			
		||||
# This file is part of cdist.
 | 
			
		||||
#
 | 
			
		||||
# cdist is free software: you can redistribute it and/or modify
 | 
			
		||||
# it under the terms of the GNU General Public License as published by
 | 
			
		||||
# the Free Software Foundation, either version 3 of the License, or
 | 
			
		||||
# (at your option) any later version.
 | 
			
		||||
#
 | 
			
		||||
# cdist is distributed in the hope that it will be useful,
 | 
			
		||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
 | 
			
		||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 | 
			
		||||
# GNU General Public License for more details.
 | 
			
		||||
#
 | 
			
		||||
# You should have received a copy of the GNU General Public License
 | 
			
		||||
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
 | 
			
		||||
#
 | 
			
		||||
#
 | 
			
		||||
 | 
			
		||||
import os
 | 
			
		||||
import shutil
 | 
			
		||||
 | 
			
		||||
import cdist
 | 
			
		||||
from cdist import core
 | 
			
		||||
from cdist import test
 | 
			
		||||
from cdist.exec import local
 | 
			
		||||
from cdist.exec import remote
 | 
			
		||||
from cdist.core import code
 | 
			
		||||
from cdist.core import manifest
 | 
			
		||||
 | 
			
		||||
import os.path as op
 | 
			
		||||
my_dir = op.abspath(op.dirname(__file__))
 | 
			
		||||
fixtures = op.join(my_dir, 'fixtures')
 | 
			
		||||
conf_dir = op.join(fixtures, 'conf')
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
class CaptureOutputDisabledTestCase(test.CdistTestCase):
 | 
			
		||||
 | 
			
		||||
    def setUp(self):
 | 
			
		||||
        # logging.root.setLevel(logging.TRACE)
 | 
			
		||||
        save_output_streams = False
 | 
			
		||||
        self.temp_dir = self.mkdtemp()
 | 
			
		||||
 | 
			
		||||
        self.local_dir = os.path.join(self.temp_dir, "local")
 | 
			
		||||
        self.hostdir = cdist.str_hash(self.target_host[0])
 | 
			
		||||
        self.host_base_path = os.path.join(self.local_dir, self.hostdir)
 | 
			
		||||
        os.makedirs(self.host_base_path)
 | 
			
		||||
        self.local = local.Local(
 | 
			
		||||
            target_host=self.target_host,
 | 
			
		||||
            target_host_tags=None,
 | 
			
		||||
            base_root_path=self.host_base_path,
 | 
			
		||||
            host_dir_name=self.hostdir,
 | 
			
		||||
            exec_path=cdist.test.cdist_exec_path,
 | 
			
		||||
            add_conf_dirs=[conf_dir],
 | 
			
		||||
            save_output_streams=save_output_streams)
 | 
			
		||||
        self.local.create_files_dirs()
 | 
			
		||||
 | 
			
		||||
        self.remote_dir = self.mkdtemp()
 | 
			
		||||
        remote_exec = self.remote_exec
 | 
			
		||||
        remote_copy = self.remote_copy
 | 
			
		||||
        self.remote = remote.Remote(
 | 
			
		||||
            target_host=self.target_host,
 | 
			
		||||
            remote_exec=remote_exec,
 | 
			
		||||
            remote_copy=remote_copy,
 | 
			
		||||
            base_path=self.remote_dir,
 | 
			
		||||
            stdout_base_path=self.local.stdout_base_path,
 | 
			
		||||
            stderr_base_path=self.local.stderr_base_path,
 | 
			
		||||
            save_output_streams=save_output_streams)
 | 
			
		||||
        self.remote.create_files_dirs()
 | 
			
		||||
 | 
			
		||||
        self.code = code.Code(self.target_host, self.local, self.remote)
 | 
			
		||||
 | 
			
		||||
        self.manifest = manifest.Manifest(self.target_host, self.local)
 | 
			
		||||
 | 
			
		||||
        self.cdist_type = core.CdistType(self.local.type_path,
 | 
			
		||||
                                         '__write_to_stdout_and_stderr')
 | 
			
		||||
        self.cdist_object = core.CdistObject(self.cdist_type,
 | 
			
		||||
                                             self.local.object_path,
 | 
			
		||||
                                             self.local.object_marker_name,
 | 
			
		||||
                                             '')
 | 
			
		||||
        self.cdist_object.create()
 | 
			
		||||
        self.output_dirs = {
 | 
			
		||||
            'object': {
 | 
			
		||||
                'stdout': os.path.join(self.cdist_object.absolute_path,
 | 
			
		||||
                                       'stdout'),
 | 
			
		||||
                'stderr': os.path.join(self.cdist_object.absolute_path,
 | 
			
		||||
                                       'stderr'),
 | 
			
		||||
            },
 | 
			
		||||
            'init': {
 | 
			
		||||
                'stdout': os.path.join(self.local.base_path, 'stdout'),
 | 
			
		||||
                'stderr': os.path.join(self.local.base_path, 'stderr'),
 | 
			
		||||
            },
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
    def tearDown(self):
 | 
			
		||||
        shutil.rmtree(self.local_dir)
 | 
			
		||||
        shutil.rmtree(self.remote_dir)
 | 
			
		||||
        shutil.rmtree(self.temp_dir)
 | 
			
		||||
 | 
			
		||||
    def _test_output(self, which, target, streams=('stdout', 'stderr')):
 | 
			
		||||
        for stream in streams:
 | 
			
		||||
            stream_path = os.path.join(self.output_dirs[target][stream], which)
 | 
			
		||||
            if os.path.exists(stream_path):
 | 
			
		||||
                with open(stream_path, 'r') as fd:
 | 
			
		||||
                    _is = fd.read()
 | 
			
		||||
                self.assertEqual("", _is)
 | 
			
		||||
            # else ok when not exists
 | 
			
		||||
 | 
			
		||||
    def test_capture_code_output_disabled(self):
 | 
			
		||||
        self.cdist_object.code_local = self.code.run_gencode_local(
 | 
			
		||||
            self.cdist_object)
 | 
			
		||||
        self._test_output('gencode-local', 'object', ('stderr',))
 | 
			
		||||
 | 
			
		||||
        self.code.run_code_local(self.cdist_object)
 | 
			
		||||
        self._test_output('code-local', 'object')
 | 
			
		||||
 | 
			
		||||
        self.cdist_object.code_remote = self.code.run_gencode_remote(
 | 
			
		||||
            self.cdist_object)
 | 
			
		||||
        self._test_output('gencode-remote', 'object', ('stderr',))
 | 
			
		||||
 | 
			
		||||
        self.code.transfer_code_remote(self.cdist_object)
 | 
			
		||||
        self.code.run_code_remote(self.cdist_object)
 | 
			
		||||
        self._test_output('code-remote', 'object')
 | 
			
		||||
 | 
			
		||||
    def test_capture_manifest_output_disabled(self):
 | 
			
		||||
        self.manifest.run_type_manifest(self.cdist_object)
 | 
			
		||||
        self._test_output('manifest', 'object')
 | 
			
		||||
 | 
			
		||||
    def test_capture_init_manifest_output_disabled(self):
 | 
			
		||||
        initial_manifest = os.path.join(conf_dir, 'manifest', 'init')
 | 
			
		||||
        self.manifest.run_initial_manifest(initial_manifest)
 | 
			
		||||
        self._test_output('init', 'init')
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
if __name__ == "__main__":
 | 
			
		||||
    import unittest
 | 
			
		||||
 | 
			
		||||
    unittest.main()
 | 
			
		||||
							
								
								
									
										4
									
								
								cdist/test/capture_output_disabled/fixtures/conf/manifest/init
									
										
									
									
									
										Executable file
									
								
							
							
						
						
									
										4
									
								
								cdist/test/capture_output_disabled/fixtures/conf/manifest/init
									
										
									
									
									
										Executable file
									
								
							| 
						 | 
				
			
			@ -0,0 +1,4 @@
 | 
			
		|||
#!/bin/sh
 | 
			
		||||
 | 
			
		||||
echo "init: stdout"
 | 
			
		||||
echo "init: stderr" >&2
 | 
			
		||||
| 
						 | 
				
			
			@ -0,0 +1,6 @@
 | 
			
		|||
#!/bin/sh
 | 
			
		||||
 | 
			
		||||
echo "gencode-local: stderr" >&2
 | 
			
		||||
 | 
			
		||||
echo "echo \"code-local: stdout\""
 | 
			
		||||
echo "echo \"code-local: stderr\" >&2"
 | 
			
		||||
| 
						 | 
				
			
			@ -0,0 +1,6 @@
 | 
			
		|||
#!/bin/sh
 | 
			
		||||
 | 
			
		||||
echo "gencode-remote: stderr" >&2
 | 
			
		||||
 | 
			
		||||
echo "echo \"code-remote: stdout\""
 | 
			
		||||
echo "echo \"code-remote: stderr\" >&2"
 | 
			
		||||
| 
						 | 
				
			
			@ -0,0 +1,4 @@
 | 
			
		|||
#!/bin/sh
 | 
			
		||||
 | 
			
		||||
echo "manifest: stdout"
 | 
			
		||||
echo "manifest: stderr" >&2
 | 
			
		||||
| 
						 | 
				
			
			@ -300,6 +300,7 @@ class ConfigurationTestCase(test.CdistTestCase):
 | 
			
		|||
        expected = {
 | 
			
		||||
            'conf_dir': ['/usr/local/cdist1', ],
 | 
			
		||||
            'verbosity': 3,
 | 
			
		||||
            'beta': False,
 | 
			
		||||
        }
 | 
			
		||||
        args_dict = vars(args)
 | 
			
		||||
        d = config._read_args_config(args_dict)
 | 
			
		||||
| 
						 | 
				
			
			@ -1167,6 +1168,118 @@ class ConfigurationTestCase(test.CdistTestCase):
 | 
			
		|||
            configuration = cc.Configuration(args, env=env,
 | 
			
		||||
                                             config_files=())
 | 
			
		||||
 | 
			
		||||
    def test_configuration_disable_saving_output_streams1(self):
 | 
			
		||||
        config = configparser.ConfigParser()
 | 
			
		||||
        config['GLOBAL'] = {
 | 
			
		||||
            'save_output_streams': 'True',
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
 | 
			
		||||
        with open(global_config_file, 'w') as f:
 | 
			
		||||
            config.write(f)
 | 
			
		||||
 | 
			
		||||
        expected_config_dict = {
 | 
			
		||||
            'GLOBAL': {
 | 
			
		||||
                'save_output_streams': True,
 | 
			
		||||
                'verbosity': 0,
 | 
			
		||||
            },
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        config_files = (global_config_file, )
 | 
			
		||||
 | 
			
		||||
        # bypass singleton so we can test further
 | 
			
		||||
        cc.Configuration.instance = None
 | 
			
		||||
 | 
			
		||||
        args = argparse.Namespace()
 | 
			
		||||
        args.save_output_streams = True
 | 
			
		||||
        configuration = cc.Configuration(args, env=None,
 | 
			
		||||
                                         config_files=config_files)
 | 
			
		||||
        self.assertEqual(configuration.config, expected_config_dict)
 | 
			
		||||
 | 
			
		||||
    def test_configuration_disable_saving_output_streams2(self):
 | 
			
		||||
        config = configparser.ConfigParser()
 | 
			
		||||
        config['GLOBAL'] = {
 | 
			
		||||
            'save_output_streams': 'False',
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
 | 
			
		||||
        with open(global_config_file, 'w') as f:
 | 
			
		||||
            config.write(f)
 | 
			
		||||
 | 
			
		||||
        expected_config_dict = {
 | 
			
		||||
            'GLOBAL': {
 | 
			
		||||
                'save_output_streams': True,
 | 
			
		||||
                'verbosity': 0,
 | 
			
		||||
            },
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        config_files = (global_config_file, )
 | 
			
		||||
 | 
			
		||||
        # bypass singleton so we can test further
 | 
			
		||||
        cc.Configuration.instance = None
 | 
			
		||||
 | 
			
		||||
        args = argparse.Namespace()
 | 
			
		||||
        args.save_output_streams = True
 | 
			
		||||
        configuration = cc.Configuration(args, env=None,
 | 
			
		||||
                                         config_files=config_files)
 | 
			
		||||
        self.assertEqual(configuration.config, expected_config_dict)
 | 
			
		||||
 | 
			
		||||
    def test_configuration_disable_saving_output_streams3(self):
 | 
			
		||||
        config = configparser.ConfigParser()
 | 
			
		||||
        config['GLOBAL'] = {
 | 
			
		||||
            'save_output_streams': 'False',
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
 | 
			
		||||
        with open(global_config_file, 'w') as f:
 | 
			
		||||
            config.write(f)
 | 
			
		||||
 | 
			
		||||
        expected_config_dict = {
 | 
			
		||||
            'GLOBAL': {
 | 
			
		||||
                'save_output_streams': False,
 | 
			
		||||
                'verbosity': 0,
 | 
			
		||||
            },
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        config_files = (global_config_file, )
 | 
			
		||||
 | 
			
		||||
        # bypass singleton so we can test further
 | 
			
		||||
        cc.Configuration.instance = None
 | 
			
		||||
 | 
			
		||||
        args = argparse.Namespace()
 | 
			
		||||
        args.save_output_streams = False
 | 
			
		||||
        configuration = cc.Configuration(args, env=None,
 | 
			
		||||
                                         config_files=config_files)
 | 
			
		||||
        self.assertEqual(configuration.config, expected_config_dict)
 | 
			
		||||
 | 
			
		||||
    def test_configuration_disable_saving_output_streams4(self):
 | 
			
		||||
        config = configparser.ConfigParser()
 | 
			
		||||
        config['GLOBAL'] = {
 | 
			
		||||
            'save_output_streams': 'True',
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        global_config_file = os.path.join(fixtures, 'cdist-global.cfg')
 | 
			
		||||
        with open(global_config_file, 'w') as f:
 | 
			
		||||
            config.write(f)
 | 
			
		||||
 | 
			
		||||
        expected_config_dict = {
 | 
			
		||||
            'GLOBAL': {
 | 
			
		||||
                'save_output_streams': False,
 | 
			
		||||
                'verbosity': 0,
 | 
			
		||||
            },
 | 
			
		||||
        }
 | 
			
		||||
 | 
			
		||||
        config_files = (global_config_file, )
 | 
			
		||||
 | 
			
		||||
        # bypass singleton so we can test further
 | 
			
		||||
        cc.Configuration.instance = None
 | 
			
		||||
 | 
			
		||||
        args = argparse.Namespace()
 | 
			
		||||
        args.save_output_streams = False
 | 
			
		||||
        configuration = cc.Configuration(args, env=None,
 | 
			
		||||
                                         config_files=config_files)
 | 
			
		||||
        self.assertEqual(configuration.config, expected_config_dict)
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
if __name__ == "__main__":
 | 
			
		||||
    import unittest
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -87,10 +87,12 @@ state
 | 
			
		|||
    this type execution state ('done' when finished)
 | 
			
		||||
 | 
			
		||||
stderr
 | 
			
		||||
  directory containing type's gencode-* and code-* stderr stream outputs
 | 
			
		||||
  directory containing type's manifest, gencode-* and code-* stderr stream
 | 
			
		||||
  outputs
 | 
			
		||||
 | 
			
		||||
stdin
 | 
			
		||||
    this type stdin content
 | 
			
		||||
 | 
			
		||||
stdout
 | 
			
		||||
  directory containing type's gencode-* and code-* stdout stream outputs.
 | 
			
		||||
  directory containing type's manifest, gencode-* and code-* stdout stream
 | 
			
		||||
  outputs.
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -88,6 +88,11 @@ The possible keywords and their meanings are as follows:
 | 
			
		|||
:strong:`remote_shell`
 | 
			
		||||
    Shell command at remote host used for remote execution.
 | 
			
		||||
 | 
			
		||||
:strong:`save_output_streams`
 | 
			
		||||
    Enable/disable saving output streams (enabled by default).
 | 
			
		||||
    It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false'
 | 
			
		||||
    and '1'/'0'.
 | 
			
		||||
 | 
			
		||||
:strong:`verbosity`
 | 
			
		||||
    Set verbosity level. Valid values are: 
 | 
			
		||||
    'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
							
								
								
									
										88
									
								
								docs/src/cdist-saving-output-streams.rst
									
										
									
									
									
										Normal file
									
								
							
							
						
						
									
										88
									
								
								docs/src/cdist-saving-output-streams.rst
									
										
									
									
									
										Normal file
									
								
							| 
						 | 
				
			
			@ -0,0 +1,88 @@
 | 
			
		|||
Saving output streams
 | 
			
		||||
=====================
 | 
			
		||||
 | 
			
		||||
Description
 | 
			
		||||
-----------
 | 
			
		||||
Since version 4.8.0 cdist, by default, saves output streams to local cache.
 | 
			
		||||
Saving output streams is implemented because important information was lost
 | 
			
		||||
during a config run, hidden in all other output.
 | 
			
		||||
Now all created output is bound to the context where it was produced.
 | 
			
		||||
 | 
			
		||||
Saving output streams include stdout and stderr of init manifest, remote
 | 
			
		||||
commands and for each object stdout and stderr of manifest, gencode-* and code-*.
 | 
			
		||||
Output stream files are created only if some output is produced. For more info
 | 
			
		||||
on these cache files see `Local cache overview <cdist-cache.html>`_.
 | 
			
		||||
 | 
			
		||||
Also, in case of an error, cdist can now exit and show all information it has
 | 
			
		||||
about the error.
 | 
			
		||||
 | 
			
		||||
For example:
 | 
			
		||||
 | 
			
		||||
.. code-block:: sh
 | 
			
		||||
 | 
			
		||||
    $ ./bin/cdist config -v -i ~/.cdist/manifest/init-output-streams $(cat ~/ungleich/data/opennebula-debian9-test )
 | 
			
		||||
    INFO: 185.203.112.42: Starting configuration run
 | 
			
		||||
    INFO: 185.203.112.42: Processing __myline/test
 | 
			
		||||
    ERROR: 185.203.112.42: Command failed: '/bin/sh -e /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-kisrqlpw/code-local'
 | 
			
		||||
    return code: 1
 | 
			
		||||
    ---- BEGIN stdout ----
 | 
			
		||||
    ---- END stdout ----
 | 
			
		||||
 | 
			
		||||
    Error processing object '__myline/test'
 | 
			
		||||
    ========================================
 | 
			
		||||
    name: __myline/test
 | 
			
		||||
    path: /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-kisrqlpw
 | 
			
		||||
    source: /home/darko/.cdist/manifest/init-output-streams
 | 
			
		||||
    type: /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/conf/type/__myline
 | 
			
		||||
 | 
			
		||||
    ---- BEGIN manifest:stderr ----
 | 
			
		||||
    myline manifest stderr
 | 
			
		||||
 | 
			
		||||
    ---- END manifest:stderr ----
 | 
			
		||||
 | 
			
		||||
    ---- BEGIN gencode-remote:stderr ----
 | 
			
		||||
    test gencode-remote error
 | 
			
		||||
 | 
			
		||||
    ---- END gencode-remote:stderr ----
 | 
			
		||||
 | 
			
		||||
    ---- BEGIN code-local:stderr ----
 | 
			
		||||
    error
 | 
			
		||||
 | 
			
		||||
    ---- END code-local:stderr ----
 | 
			
		||||
 | 
			
		||||
    ERROR: cdist: Failed to configure the following hosts: 185.203.112.42
 | 
			
		||||
 | 
			
		||||
Upon successful run execution state is saved to local cache and temporary
 | 
			
		||||
directory is removed.
 | 
			
		||||
In case of an error temporary directory is not removed and can be further
 | 
			
		||||
discovered.
 | 
			
		||||
 | 
			
		||||
There is also an option :strong:`-S/--disable-saving-output-streams` for
 | 
			
		||||
disabling saving output streams. In this case error reporting can look
 | 
			
		||||
like this:
 | 
			
		||||
 | 
			
		||||
.. code-block:: sh
 | 
			
		||||
 | 
			
		||||
    $ ./bin/cdist config -v -S -i ~/.cdist/manifest/init-output-streams $(cat ~/ungleich/data/opennebula-debian9-test )
 | 
			
		||||
    INFO: 185.203.112.42: Starting configuration run
 | 
			
		||||
    test stdout output streams
 | 
			
		||||
    test stderr output streams
 | 
			
		||||
    myline manifest stdout
 | 
			
		||||
    myline manifest stderr
 | 
			
		||||
    test gencode-remote error
 | 
			
		||||
    INFO: 185.203.112.42: Processing __myline/test
 | 
			
		||||
    error
 | 
			
		||||
    ERROR: 185.203.112.42: Command failed: '/bin/sh -e /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-n566pqut/code-local'
 | 
			
		||||
    return code: 1
 | 
			
		||||
    ---- BEGIN stdout ----
 | 
			
		||||
    ---- END stdout ----
 | 
			
		||||
 | 
			
		||||
    Error processing object '__myline/test'
 | 
			
		||||
    ========================================
 | 
			
		||||
    name: __myline/test
 | 
			
		||||
    path: /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-n566pqut
 | 
			
		||||
    source: /home/darko/.cdist/manifest/init-output-streams
 | 
			
		||||
    type: /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/conf/type/__myline
 | 
			
		||||
 | 
			
		||||
 | 
			
		||||
    ERROR: cdist: Failed to configure the following hosts: 185.203.112.42
 | 
			
		||||
| 
						 | 
				
			
			@ -31,6 +31,7 @@ Contents:
 | 
			
		|||
   cdist-best-practice
 | 
			
		||||
   cdist-stages
 | 
			
		||||
   cdist-cache
 | 
			
		||||
   cdist-saving-output-streams
 | 
			
		||||
   cdist-remote-exec-copy
 | 
			
		||||
   cdist-hacker
 | 
			
		||||
   cdist-troubleshooting
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
| 
						 | 
				
			
			@ -21,7 +21,7 @@ SYNOPSIS
 | 
			
		|||
                 [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
 | 
			
		||||
                 [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
 | 
			
		||||
                 [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
 | 
			
		||||
                 [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
 | 
			
		||||
                 [-f HOSTFILE] [-p [HOST_MAX]] [-S] [-s] [-t]
 | 
			
		||||
                 [host [host ...]] 
 | 
			
		||||
 | 
			
		||||
    cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
 | 
			
		||||
| 
						 | 
				
			
			@ -29,7 +29,7 @@ SYNOPSIS
 | 
			
		|||
                  [-j [JOBS]] [-n] [-o OUT_PATH] [-R [{tar,tgz,tbz2,txz}]]
 | 
			
		||||
                  [-r REMOTE_OUT_DIR] [--remote-copy REMOTE_COPY]
 | 
			
		||||
                  [--remote-exec REMOTE_EXEC] [-I INVENTORY_DIR] [-A] [-a]
 | 
			
		||||
                  [-f HOSTFILE] [-p [HOST_MAX]] [-s] [-t]
 | 
			
		||||
                  [-f HOSTFILE] [-p [HOST_MAX]] [-S] [-s] [-t]
 | 
			
		||||
                  [host [host ...]] 
 | 
			
		||||
 | 
			
		||||
    cdist inventory [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE]
 | 
			
		||||
| 
						 | 
				
			
			@ -200,6 +200,10 @@ Install command is currently in beta.
 | 
			
		|||
 | 
			
		||||
    Directory to save cdist output in on the target host.
 | 
			
		||||
 | 
			
		||||
.. option:: -S, --disable-saving-output-streams
 | 
			
		||||
 | 
			
		||||
    Disable saving output streams.
 | 
			
		||||
 | 
			
		||||
.. option:: -s, --sequential
 | 
			
		||||
 | 
			
		||||
    Operate on multiple hosts sequentially (default).
 | 
			
		||||
| 
						 | 
				
			
			@ -561,6 +565,11 @@ The possible keywords and their meanings are as follows:
 | 
			
		|||
:strong:`remote_shell`
 | 
			
		||||
    Shell command at remote host used for remote execution.
 | 
			
		||||
 | 
			
		||||
:strong:`save_output_streams`
 | 
			
		||||
    Enable/disable saving output streams (enabled by default).
 | 
			
		||||
    It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false'
 | 
			
		||||
    and '1'/'0'.
 | 
			
		||||
 | 
			
		||||
:strong:`verbosity`
 | 
			
		||||
    Set verbosity level. Valid values are: 
 | 
			
		||||
    'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'.
 | 
			
		||||
| 
						 | 
				
			
			
 | 
			
		|||
		Loading…
	
	Add table
		Add a link
		
	
		Reference in a new issue