From 0e8302c20403da8a487b0ba5363741a14f01c079 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 4 Jul 2017 19:23:51 +0300 Subject: [PATCH 01/23] if no --source, then create only if there's no file --- cdist/conf/type/__file/gencode-local | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index c79fc16a..932251fe 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -29,8 +29,11 @@ upload_file= create_file= if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then if [ ! -f "$__object/parameter/source" ]; then - create_file=1 - echo create >> "$__messages_out" + remote_stat="$(cat "$__object/explorer/stat")" + if [ -z "$remote_stat" ]; then + create_file=1 + echo create >> "$__messages_out" + fi else source="$(cat "$__object/parameter/source")" if [ "$source" = "-" ]; then From 6a6716035bfd52043ac34621ab49b3019fb30fe8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Tue, 4 Jul 2017 19:25:08 +0300 Subject: [PATCH 02/23] one space too much --- cdist/conf/type/__file/gencode-local | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 932251fe..aec4e43e 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -31,8 +31,8 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then if [ ! -f "$__object/parameter/source" ]; then remote_stat="$(cat "$__object/explorer/stat")" if [ -z "$remote_stat" ]; then - create_file=1 - echo create >> "$__messages_out" + create_file=1 + echo create >> "$__messages_out" fi else source="$(cat "$__object/parameter/source")" From 01ca5910ecb463dcb1e546ce26e499e0614afeb9 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Jul 2017 23:49:27 +0200 Subject: [PATCH 03/23] increase ControlPersist to 30 to workaround #542 Signed-off-by: Steven Armstrong --- cdist/util/remoteutil.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/util/remoteutil.py b/cdist/util/remoteutil.py index 2bd12fdf..4950ef8b 100644 --- a/cdist/util/remoteutil.py +++ b/cdist/util/remoteutil.py @@ -36,7 +36,7 @@ def inspect_ssh_mux_opts(): wanted_mux_opts = { "ControlPath": "{}", "ControlMaster": "auto", - "ControlPersist": "10", + "ControlPersist": "30", } mux_opts = " ".join([" -o {}={}".format( x, wanted_mux_opts[x]) for x in wanted_mux_opts]) From c63ab44c9cefcedd355d6952677d3cdab9ac61fe Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 28 Jun 2017 10:18:53 +0200 Subject: [PATCH 04/23] Define better warning facility. --- cdist/__init__.py | 2 +- cdist/argparse.py | 31 +++++++++++++++++++++++-------- cdist/log.py | 35 +++++++++++++++++++++++++++++++++++ cdist/util/ipaddr.py | 4 ++-- docs/src/man1/cdist.rst | 10 +++++++--- scripts/cdist | 3 --- 6 files changed, 68 insertions(+), 17 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index b0fd75ea..d92205a6 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -20,9 +20,9 @@ # import os -import subprocess import hashlib +import cdist.log import cdist.version VERSION = cdist.version.VERSION diff --git a/cdist/argparse.py b/cdist/argparse.py index 3accedeb..8689be79 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -17,13 +17,20 @@ EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" parser = None +_verbosity_level_off = -2 +_verbosity_level_debug = 3 _verbosity_level = { - 0: logging.ERROR, - 1: logging.WARNING, - 2: logging.INFO, + _verbosity_level_off: logging.OFF, + -1: logging.ERROR, + 0: logging.WARNING, + 1: logging.INFO, + 2: logging.VERBOSE, + 3: logging.DEBUG, + 4: logging.TRACE, } +# All verbosity levels above 4 are TRACE. _verbosity_level = collections.defaultdict( - lambda: logging.DEBUG, _verbosity_level) + lambda: logging.TRACE, _verbosity_level) def add_beta_command(cmd): @@ -83,13 +90,18 @@ def get_parsers(): '-d', '--debug', help=('Set log level to debug (deprecated, use -vvv instead)'), action='store_true', default=False) + parser['loglevel'].add_argument( + '-q', '--quiet', + help='Quiet mode: disables logging, including WARNING and ERROR', + action='store_true', default=False) parser['loglevel'].add_argument( '-v', '--verbose', help=('Increase the verbosity level. Every instance of -v ' 'increments the verbosity level by one. Its default value ' - 'is 0. There are 4 levels of verbosity. The order of levels ' - 'from the lowest to the highest are: ERROR (0), ' - 'WARNING (1), INFO (2) and DEBUG (3 or higher).'), + 'is 0 which includes ERROR and WARNING levels. ' + 'The levels, in order from the lowest to the highest, are: ' + 'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) ' + 'TRACE (4 or higher).'), action='count', default=0) parser['beta'] = argparse.ArgumentParser(add_help=False) @@ -213,10 +225,13 @@ def get_parsers(): def handle_loglevel(args): if args.debug: retval = "-d/--debug is deprecated, use -vvv instead" - args.verbose = 3 + args.verbose = _verbosity_level_debug else: retval = None + if args.quiet: + args.verbose = _verbosity_level_off + logging.root.setLevel(_verbosity_level[args.verbose]) return retval diff --git a/cdist/log.py b/cdist/log.py index 2341c282..ce0addcc 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -23,6 +23,31 @@ import logging +# Define additional cdist logging levels. +logging.OFF = logging.CRITICAL + 10 # disable logging +logging.addLevelName(logging.OFF, 'OFF') + +logging.VERBOSE = logging.INFO - 5 +logging.addLevelName(logging.VERBOSE, 'VERBOSE') + + +def _verbose(msg, *args, **kwargs): + logging.log(logging.VERBOSE, msg, *args, **kwargs) + + +logging.verbose = _verbose + +logging.TRACE = logging.DEBUG - 5 +logging.addLevelName(logging.TRACE, 'TRACE') + + +def _trace(msg, *args, **kwargs): + logging.log(logging.TRACE, msg, *args, **kwargs) + + +logging.trace = _trace + + class Log(logging.Logger): def __init__(self, name): @@ -37,3 +62,13 @@ class Log(logging.Logger): record.msg = self.name + ": " + str(record.msg) return True + + def verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, *args, **kwargs) + + def trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) + + +logging.setLoggerClass(Log) +logging.basicConfig(format='%(levelname)s: %(message)s') diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index a747aeb5..b9d1ab30 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -45,7 +45,7 @@ def resolve_target_host_name(host): log.debug("derived host_name for host \"{}\": {}".format( host, host_name)) except (socket.gaierror, socket.herror) as e: - log.warn("Could not derive host_name for {}" + log.warning("Could not derive host_name for {}" ", $host_name will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_name = '' @@ -59,7 +59,7 @@ def resolve_target_fqdn(host): log.debug("derived host_fqdn for host \"{}\": {}".format( host, host_fqdn)) except socket.herror as e: - log.warn("Could not derive host_fqdn for {}" + log.warning("Could not derive host_fqdn for {}" ", $host_fqdn will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_fqdn = '' diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 3bddbd02..dbfda34f 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -50,12 +50,16 @@ All commands accept the following options: Set log level to debug (deprecated, use -vvv instead) +.. option:: -q, --quiet + + Quiet mode: disables logging, including WARNING and ERROR + .. option:: -v, --verbose Increase the verbosity level. Every instance of -v increments the verbosity - level by one. Its default value is 0. There are 4 levels of verbosity. The - order of levels from the lowest to the highest are: ERROR (0), WARNING (1), - INFO (2) and DEBUG (3 or higher). + level by one. Its default value is 0 which includes ERROR and WARNING levels. + The levels, in order from the lowest to the highest, are: + ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4 or higher). .. option:: -V, --version diff --git a/scripts/cdist b/scripts/cdist index 498091b8..9719e6ee 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -80,10 +80,7 @@ if __name__ == "__main__": import os import re import cdist - import cdist.log - logging.setLoggerClass(cdist.log.Log) - logging.basicConfig(format='%(levelname)s: %(message)s') log = logging.getLogger("cdist") if re.match("__", os.path.basename(sys.argv[0])): From 248656b81fe761626085daec7bafb0354cfca811 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 28 Jun 2017 23:36:42 +0200 Subject: [PATCH 05/23] First iteration of logging cleanup. --- cdist/config.py | 24 ++++++++++++------------ cdist/core/explorer.py | 20 ++++++++++---------- cdist/core/manifest.py | 2 +- cdist/emulator.py | 6 +++--- cdist/exec/local.py | 18 +++++++++--------- cdist/exec/remote.py | 20 ++++++++++---------- cdist/shell.py | 4 ++-- scripts/cdist | 4 ++-- 8 files changed, 49 insertions(+), 49 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 19679a4c..f6b0cc20 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -144,7 +144,7 @@ class Config(object): hostcnt += 1 if args.parallel: - log.debug("Creating child process for %s", host) + log.trace("Creating child process for %s", host) process[host] = multiprocessing.Process( target=cls.onehost, args=(host, host_base_path, hostdir, args, True)) @@ -159,7 +159,7 @@ class Config(object): # Catch errors in parallel mode when joining if args.parallel: for host in process.keys(): - log.debug("Joining process %s", host) + log.trace("Joining process %s", host) process[host].join() if not process[host].exitcode == 0: @@ -300,7 +300,7 @@ class Config(object): return objects_changed def _iterate_once_sequential(self): - self.log.info("Iteration in sequential mode") + self.log.debug("Iteration in sequential mode") objects_changed = False for cdist_object in self.object_list(): @@ -327,7 +327,7 @@ class Config(object): return objects_changed def _iterate_once_parallel(self): - self.log.info("Iteration in parallel mode in {} jobs".format( + self.log.debug("Iteration in parallel mode in {} jobs".format( self.jobs)) objects_changed = False @@ -350,15 +350,15 @@ class Config(object): self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for {} parallel " + self.log.trace(("Starting multiprocessing Pool for {} parallel " "objects preparation".format(n))) args = [ (c, ) for c in cargo ] mp_pool_run(self.object_prepare, args, jobs=self.jobs) - self.log.debug(("Multiprocessing for parallel object " + self.log.trace(("Multiprocessing for parallel object " "preparation finished")) objects_changed = True @@ -386,15 +386,15 @@ class Config(object): self.object_run(cargo[0]) objects_changed = True elif cargo: - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for {} parallel " + self.log.trace(("Starting multiprocessing Pool for {} parallel " "object run".format(n))) args = [ (c, ) for c in cargo ] mp_pool_run(self.object_run, args, jobs=self.jobs) - self.log.debug(("Multiprocessing for parallel object " + self.log.trace(("Multiprocessing for parallel object " "run finished")) objects_changed = True @@ -473,7 +473,7 @@ class Config(object): def object_run(self, cdist_object): """Run gencode and code for an object""" - self.log.debug("Trying to run object %s" % (cdist_object.name)) + self.log.verbose("Trying to run object %s" % (cdist_object.name)) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " "object: %s"), cdist_object) @@ -500,5 +500,5 @@ class Config(object): self.log.info("Skipping code execution due to DRY RUN") # Mark this object as done - self.log.debug("Finishing run of " + cdist_object.name) + self.log.trace("Finishing run of " + cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 45afc5c0..aea17d83 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -95,7 +95,7 @@ class Explorer(object): out_path directory. """ - self.log.info("Running global explorers") + self.log.trace("Running global explorers") self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) @@ -109,22 +109,22 @@ class Explorer(object): fd.write(output) def _run_global_explorers_seq(self, out_path): - self.log.info("Running global explorers sequentially") + self.log.debug("Running global explorers sequentially") for explorer in self.list_global_explorer_names(): self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.info("Running global explorers in {} parallel jobs".format( + self.log.debug("Running global explorers in {} parallel jobs".format( self.jobs)) - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for global " + self.log.trace(("Starting multiprocessing Pool for global " "explorers run")) args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.debug(("Multiprocessing run for global explorers " + self.log.trace(("Multiprocessing run for global explorers " "finished")) # logger is not pickable, so remove it when we pickle @@ -168,15 +168,15 @@ class Explorer(object): in the object. """ - self.log.debug("Transfering type explorers for type: %s", + self.log.trace("Transfering type explorers for type: %s", cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) - self.log.debug("Transfering object parameters for object: %s", + self.log.trace("Transfering object parameters for object: %s", cdist_object.name) self.transfer_object_parameters(cdist_object) for explorer in self.list_type_explorer_names(cdist_object.cdist_type): output = self.run_type_explorer(explorer, cdist_object) - self.log.debug("Running type explorer '%s' for object '%s'", + self.log.trace("Running type explorer '%s' for object '%s'", explorer, cdist_object.name) cdist_object.explorers[explorer] = output @@ -203,7 +203,7 @@ class Explorer(object): remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.debug("Skipping retransfer of type explorers for: %s", + self.log.trace("Skipping retransfer of type explorers for: %s", cdist_type) else: source = os.path.join(self.local.type_path, diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 29f96c4f..cd0fde9c 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -145,7 +145,7 @@ class Manifest(object): else: user_supplied = True - self.log.info("Running initial manifest " + initial_manifest) + self.log.trace("Running initial manifest " + initial_manifest) if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) diff --git a/cdist/emulator.py b/cdist/emulator.py index cdbe5b08..09813d64 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -98,7 +98,7 @@ class Emulator(object): self.save_stdin() self.record_requirements() self.record_auto_requirements() - self.log.debug("Finished %s %s" % ( + self.log.trace("Finished %s %s" % ( self.cdist_object.path, self.parameters)) def __init_log(self): @@ -148,7 +148,7 @@ class Emulator(object): # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.debug('Args: %s' % self.args) + self.log.trace('Args: %s' % self.args) def setup_object(self): # Setup object - and ensure it is not in args @@ -240,7 +240,7 @@ class Emulator(object): self.object_source))) raise - self.log.debug("Recording requirement: %s", requirement) + self.log.trace("Recording requirement: %s", requirement) # Save the sanitised version, not the user supplied one # (__file//bar => __file/bar) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index fc2e7c09..be3cde4c 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -163,7 +163,7 @@ class Local(object): with open(self.object_marker_file, 'w') as fd: fd.write("%s\n" % self.object_marker_name) - self.log.debug("Object marker %s saved in %s" % ( + self.log.trace("Object marker %s saved in %s" % ( self.object_marker_name, self.object_marker_file)) def _init_cache_dir(self, cache_dir): @@ -178,12 +178,12 @@ class Local(object): def rmdir(self, path): """Remove directory on the local side.""" - self.log.debug("Local rmdir: %s", path) + self.log.trace("Local rmdir: %s", path) shutil.rmtree(path) def mkdir(self, path): """Create directory on the local side.""" - self.log.debug("Local mkdir: %s", path) + self.log.trace("Local mkdir: %s", path) os.makedirs(path, exist_ok=True) def run(self, command, env=None, return_output=False, message_prefix=None, @@ -192,7 +192,7 @@ class Local(object): Return the output as a string. """ - self.log.debug("Local run: %s", command) + self.log.trace("Local run: %s", command) assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: %s" % command) @@ -214,9 +214,9 @@ class Local(object): try: if save_output: output, errout = exec_util.call_get_output(command, env=env) - self.log.debug("Local stdout: {}".format(output)) + self.log.trace("Local stdout: {}".format(output)) # Currently, stderr is not captured. - # self.log.debug("Local stderr: {}".format(errout)) + # self.log.trace("Local stderr: {}".format(errout)) if return_output: return output.decode() else: @@ -279,7 +279,7 @@ class Local(object): return cache_subpath def save_cache(self, start_time=time.time()): - self.log.debug("cache subpath pattern: {}".format( + self.log.trace("cache subpath pattern: {}".format( self.cache_path_pattern)) cache_subpath = self._cache_subpath(start_time, self.cache_path_pattern) @@ -340,7 +340,7 @@ class Local(object): if os.path.exists(dst): os.unlink(dst) - self.log.debug("Linking %s to %s ..." % (src, dst)) + self.log.trace("Linking %s to %s ..." % (src, dst)) try: os.symlink(src, dst) except OSError as e: @@ -352,7 +352,7 @@ class Local(object): src = os.path.abspath(self.exec_path) for cdist_type in core.CdistType.list_types(self.type_path): dst = os.path.join(self.bin_path, cdist_type.name) - self.log.debug("Linking emulator: %s to %s", src, dst) + self.log.trace("Linking emulator: %s to %s", src, dst) try: os.symlink(src, dst) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 042b7103..ff4ad30a 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -111,17 +111,17 @@ class Remote(object): def rmdir(self, path): """Remove directory on the remote side.""" - self.log.debug("Remote rmdir: %s", path) + self.log.trace("Remote rmdir: %s", path) self.run(["rm", "-rf", path]) def mkdir(self, path): """Create directory on the remote side.""" - self.log.debug("Remote mkdir: %s", path) + self.log.trace("Remote mkdir: %s", path) self.run(["mkdir", "-p", path]) def transfer(self, source, destination, jobs=None): """Transfer a file or directory to the remote side.""" - self.log.debug("Remote transfer: %s -> %s", source, destination) + self.log.trace("Remote transfer: %s -> %s", source, destination) self.rmdir(destination) if os.path.isdir(source): self.mkdir(destination) @@ -147,11 +147,11 @@ class Remote(object): def _transfer_dir_parallel(self, source, destination, jobs): """Transfer a directory to the remote side in parallel mode.""" - self.log.info("Remote transfer in {} parallel jobs".format( + self.log.debug("Remote transfer in {} parallel jobs".format( jobs)) - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for parallel " + self.log.trace(("Starting multiprocessing Pool for parallel " "remote transfer")) args = [] for f in glob.glob1(source, '*'): @@ -161,7 +161,7 @@ class Remote(object): _wrap_addr(self.target_host[0]), destination)]) args.append((command, )) mp_pool_run(self._run_command, args, jobs=jobs) - self.log.debug(("Multiprocessing for parallel transfer " + self.log.trace(("Multiprocessing for parallel transfer " "finished")) def run_script(self, script, env=None, return_output=False): @@ -226,12 +226,12 @@ class Remote(object): os_environ['__target_hostname'] = self.target_host[1] os_environ['__target_fqdn'] = self.target_host[2] - self.log.debug("Remote run: %s", command) + self.log.trace("Remote run: %s", command) try: output, errout = exec_util.call_get_output(command, env=os_environ) - self.log.debug("Remote stdout: {}".format(output)) + self.log.trace("Remote stdout: {}".format(output)) # Currently, stderr is not captured. - # self.log.debug("Remote stderr: {}".format(errout)) + # self.log.trace("Remote stderr: {}".format(errout)) if return_output: return output.decode() except subprocess.CalledProcessError as e: diff --git a/cdist/shell.py b/cdist/shell.py index 9378efc3..662f8f7d 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -86,10 +86,10 @@ class Shell(object): self._init_files_dirs() self._init_environment() - log.info("Starting shell...") + log.trace("Starting shell...") # save_output=False -> do not catch stdout and stderr self.local.run([self.shell], self.env, save_output=False) - log.info("Finished shell.") + log.trace("Finished shell.") @classmethod def commandline(cls, args): diff --git a/scripts/cdist b/scripts/cdist index 9719e6ee..492bbec6 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -44,8 +44,8 @@ def commandline(): if retval: log.warning(retval) - log.debug(args) - log.info("version %s" % cdist.VERSION) + log.trace(args) + log.verbose("version %s" % cdist.VERSION) # Work around python 3.3 bug: # http://bugs.python.org/issue16308 From 127c05a882e79c50822ff4ad1226d8ca532b047d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 28 Jun 2017 23:43:39 +0200 Subject: [PATCH 06/23] 2nd iteration of logging cleanup. --- cdist/core/explorer.py | 2 +- cdist/core/manifest.py | 2 +- cdist/emulator.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index aea17d83..f2f1f15b 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -95,7 +95,7 @@ class Explorer(object): out_path directory. """ - self.log.trace("Running global explorers") + self.log.info("Running global explorers") self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index cd0fde9c..29f96c4f 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -145,7 +145,7 @@ class Manifest(object): else: user_supplied = True - self.log.trace("Running initial manifest " + initial_manifest) + self.log.info("Running initial manifest " + initial_manifest) if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) diff --git a/cdist/emulator.py b/cdist/emulator.py index 09813d64..78d75b26 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -240,7 +240,7 @@ class Emulator(object): self.object_source))) raise - self.log.trace("Recording requirement: %s", requirement) + self.log.debug("Recording requirement: %s", requirement) # Save the sanitised version, not the user supplied one # (__file//bar => __file/bar) From a722f3c63486d1f3ed4f93727f935a3b8dab3731 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 29 Jun 2017 09:22:59 +0200 Subject: [PATCH 07/23] 3rd iteration of logging cleanup. --- cdist/config.py | 8 +++++++- cdist/core/cdist_object.py | 3 --- cdist/core/cdist_type.py | 1 - cdist/core/code.py | 3 --- cdist/core/explorer.py | 2 ++ cdist/core/manifest.py | 4 ++-- cdist/emulator.py | 4 ++-- cdist/exec/local.py | 2 +- scripts/cdist | 2 +- 9 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index f6b0cc20..4109fa85 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -464,6 +464,7 @@ class Config(object): def object_prepare(self, cdist_object): """Prepare object: Run type explorer + manifest""" + self.log.info("Preparing object {}".format(cdist_object.name)) self.log.info( "Running manifest and explorers for " + cdist_object.name) self.explorer.run_type_explorers(cdist_object) @@ -473,6 +474,7 @@ class Config(object): def object_run(self, cdist_object): """Run gencode and code for an object""" + self.log.info("Running object " + cdist_object.name) self.log.verbose("Trying to run object %s" % (cdist_object.name)) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " @@ -481,7 +483,7 @@ class Config(object): cdist_type = cdist_object.cdist_type # Generate - self.log.info("Generating code for %s" % (cdist_object.name)) + self.log.debug("Generating code for %s" % (cdist_object.name)) cdist_object.code_local = self.code.run_gencode_local(cdist_object) cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) if cdist_object.code_local or cdist_object.code_remote: @@ -492,8 +494,12 @@ class Config(object): if cdist_object.code_local or cdist_object.code_remote: self.log.info("Executing code for %s" % (cdist_object.name)) if cdist_object.code_local: + self.log.trace("Executing local code for %s" + % (cdist_object.name)) self.code.run_code_local(cdist_object) if cdist_object.code_remote: + self.log.trace("Executing remote code for %s" + % (cdist_object.name)) self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) else: diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 262db8bf..410dfafc 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -22,7 +22,6 @@ # import fnmatch -import logging import os import collections @@ -30,8 +29,6 @@ import cdist import cdist.core from cdist.util import fsproperty -log = logging.getLogger(__name__) - class IllegalObjectIdError(cdist.Error): def __init__(self, object_id, message=None): diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 14865386..339d5995 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -21,7 +21,6 @@ # import os - import cdist diff --git a/cdist/core/code.py b/cdist/core/code.py index e9e2edf0..6917e7ba 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -21,13 +21,10 @@ # # -import logging import os import cdist -log = logging.getLogger(__name__) - ''' common: diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index f2f1f15b..b5c00897 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -168,6 +168,8 @@ class Explorer(object): in the object. """ + self.log.info("Running type explorers for {}".format( + cdist_object.cdist_type)) self.log.trace("Transfering type explorers for type: %s", cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 29f96c4f..16c08353 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -145,12 +145,11 @@ class Manifest(object): else: user_supplied = True - self.log.info("Running initial manifest " + initial_manifest) - if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" + self.log.info("Running initial manifest " + initial_manifest) self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix, @@ -177,6 +176,7 @@ class Manifest(object): cdist_object.cdist_type.manifest_path) message_prefix = cdist_object.name if os.path.isfile(type_manifest): + self.log.info("Running type manifest " + type_manifest) self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix, diff --git a/cdist/emulator.py b/cdist/emulator.py index 78d75b26..7c9dfcca 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -256,10 +256,10 @@ class Emulator(object): # (this would leed to an circular dependency) if ("CDIST_ORDER_DEPENDENCY" in self.env and 'CDIST_OVERRIDE' not in self.env): - # load object name created bevor this one from typeorder file ... + # load object name created befor this one from typeorder file ... with open(self.typeorder_path, 'r') as typecreationfile: typecreationorder = typecreationfile.readlines() - # get the type created bevore this one ... + # get the type created before this one ... try: lastcreatedtype = typecreationorder[-2].strip() if 'require' in self.env: diff --git a/cdist/exec/local.py b/cdist/exec/local.py index be3cde4c..2ff31466 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -192,7 +192,6 @@ class Local(object): Return the output as a string. """ - self.log.trace("Local run: %s", command) assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: %s" % command) @@ -211,6 +210,7 @@ class Local(object): message = cdist.message.Message(message_prefix, self.messages_path) env.update(message.env) + self.log.trace("Local run: %s", command) try: if save_output: output, errout = exec_util.call_get_output(command, env=env) diff --git a/scripts/cdist b/scripts/cdist index 492bbec6..605c5a6c 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -45,7 +45,7 @@ def commandline(): log.warning(retval) log.trace(args) - log.verbose("version %s" % cdist.VERSION) + log.info("version %s" % cdist.VERSION) # Work around python 3.3 bug: # http://bugs.python.org/issue16308 From a37d286d6759aeb8d58943da97d28e5eb5da7701 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 29 Jun 2017 10:18:46 +0200 Subject: [PATCH 08/23] Suppress subprocess script output in quiet mode. --- cdist/config.py | 6 ++++-- cdist/exec/local.py | 18 +++++++++++++++--- cdist/exec/remote.py | 11 +++++++++-- cdist/exec/util.py | 8 ++++---- 4 files changed, 32 insertions(+), 11 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 4109fa85..b17ec67f 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -220,13 +220,15 @@ class Config(object): host_dir_name=host_dir_name, initial_manifest=args.manifest, add_conf_dirs=args.conf_dir, - cache_path_pattern=args.cache_path_pattern) + cache_path_pattern=args.cache_path_pattern, + quiet_mode=args.quiet) remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, remote_copy=remote_copy, - base_path=args.remote_out_path) + base_path=args.remote_out_path, + quiet_mode=args.quiet) c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs) c.run() diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 2ff31466..06127660 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -54,7 +54,8 @@ class Local(object): exec_path=sys.argv[0], initial_manifest=None, add_conf_dirs=None, - cache_path_pattern=None): + cache_path_pattern=None, + quiet_mode=False): self.target_host = target_host self.hostdir = host_dir_name @@ -64,6 +65,7 @@ class Local(object): self.custom_initial_manifest = initial_manifest self._add_conf_dirs = add_conf_dirs self.cache_path_pattern = cache_path_pattern + self.quiet_mode = quiet_mode self._init_log() self._init_permissions() @@ -212,8 +214,13 @@ class Local(object): self.log.trace("Local run: %s", command) try: + if self.quiet_mode: + stderr = subprocess.DEVNULL + else: + stderr = None if save_output: - output, errout = exec_util.call_get_output(command, env=env) + output, errout = exec_util.call_get_output( + command, env=env, stderr=stderr) self.log.trace("Local stdout: {}".format(output)) # Currently, stderr is not captured. # self.log.trace("Local stderr: {}".format(errout)) @@ -223,7 +230,12 @@ class Local(object): # In some cases no output is saved. # This is used for shell command, stdout and stderr # must not be catched. - subprocess.check_call(command, env=env) + if self.quiet_mode: + stdout = subprocess.DEVNULL + else: + stdout = None + subprocess.check_call(command, env=env, stderr=stderr, + stdout=stdout) except subprocess.CalledProcessError as e: exec_util.handle_called_process_error(e, command) except OSError as error: diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index ff4ad30a..2a27bae4 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -62,7 +62,8 @@ class Remote(object): target_host, remote_exec, remote_copy, - base_path=None): + base_path=None, + quiet_mode=None): self.target_host = target_host self._exec = remote_exec self._copy = remote_copy @@ -71,6 +72,7 @@ class Remote(object): self.base_path = base_path else: self.base_path = "/var/lib/cdist" + self.quiet_mode = quiet_mode self.conf_path = os.path.join(self.base_path, "conf") self.object_path = os.path.join(self.base_path, "object") @@ -228,7 +230,12 @@ class Remote(object): self.log.trace("Remote run: %s", command) try: - output, errout = exec_util.call_get_output(command, env=os_environ) + if self.quiet_mode: + stderr = subprocess.DEVNULL + else: + stderr = None + output, errout = exec_util.call_get_output( + command, env=os_environ, stderr=stderr) self.log.trace("Remote stdout: {}".format(output)) # Currently, stderr is not captured. # self.log.trace("Remote stderr: {}".format(errout)) diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 864a73a3..9ed7103b 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -116,14 +116,14 @@ import cdist # return (result.stdout, result.stderr) -def call_get_output(command, env=None): +def call_get_output(command, env=None, stderr=None): """Run the given command with the given environment. Return the tuple of stdout and stderr output as a byte strings. """ assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: {}".format(command)) - return (_call_get_stdout(command, env), None) + return (_call_get_stdout(command, env, stderr), None) def handle_called_process_error(err, command): @@ -140,7 +140,7 @@ def handle_called_process_error(err, command): err.returncode, err.output)) -def _call_get_stdout(command, env=None): +def _call_get_stdout(command, env=None, stderr=None): """Run the given command with the given environment. Return the stdout output as a byte string, stderr is ignored. """ @@ -148,7 +148,7 @@ def _call_get_stdout(command, env=None): "list or tuple argument expected, got: {}".format(command)) with TemporaryFile() as fout: - subprocess.check_call(command, env=env, stdout=fout) + subprocess.check_call(command, env=env, stdout=fout, stderr=stderr) fout.seek(0) output = fout.read() From 015861e63bc7d0a9ad9b246d14d8fa35dd9c9abf Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 29 Jun 2017 14:46:35 +0200 Subject: [PATCH 09/23] pep8 --- cdist/exec/local.py | 3 ++- cdist/util/ipaddr.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 06127660..6c285204 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -297,7 +297,8 @@ class Local(object): self.cache_path_pattern) self.log.debug("cache subpath: {}".format(cache_subpath)) destination = os.path.join(self.cache_path, cache_subpath) - self.log.debug("Saving " + self.base_path + " to " + destination) + self.log.trace(("Saving cache: " + self.base_path + " to " + + destination)) if not os.path.exists(destination): shutil.move(self.base_path, destination) diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index b9d1ab30..0bcb0a83 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -46,7 +46,7 @@ def resolve_target_host_name(host): host, host_name)) except (socket.gaierror, socket.herror) as e: log.warning("Could not derive host_name for {}" - ", $host_name will be empty. Error is: {}".format(host, e)) + ", $host_name will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_name = '' return host_name @@ -60,7 +60,7 @@ def resolve_target_fqdn(host): host, host_fqdn)) except socket.herror as e: log.warning("Could not derive host_fqdn for {}" - ", $host_fqdn will be empty. Error is: {}".format(host, e)) + ", $host_fqdn will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_fqdn = '' return host_fqdn From cef1d215b265ff4f9cba1406d6ad08062387c02c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 30 Jun 2017 09:08:22 +0200 Subject: [PATCH 10/23] Finally remove deprecated -d/--debug option. --- cdist/argparse.py | 13 ------------- docs/src/man1/cdist.rst | 18 +++++++----------- scripts/cdist | 6 ++---- 3 files changed, 9 insertions(+), 28 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 8689be79..8208d1f0 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -18,7 +18,6 @@ parser = None _verbosity_level_off = -2 -_verbosity_level_debug = 3 _verbosity_level = { _verbosity_level_off: logging.OFF, -1: logging.ERROR, @@ -86,10 +85,6 @@ def get_parsers(): parser = {} # Options _all_ parsers have in common parser['loglevel'] = argparse.ArgumentParser(add_help=False) - parser['loglevel'].add_argument( - '-d', '--debug', - help=('Set log level to debug (deprecated, use -vvv instead)'), - action='store_true', default=False) parser['loglevel'].add_argument( '-q', '--quiet', help='Quiet mode: disables logging, including WARNING and ERROR', @@ -223,15 +218,7 @@ def get_parsers(): def handle_loglevel(args): - if args.debug: - retval = "-d/--debug is deprecated, use -vvv instead" - args.verbose = _verbosity_level_debug - else: - retval = None - if args.quiet: args.verbose = _verbosity_level_off logging.root.setLevel(_verbosity_level[args.verbose]) - - return retval diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index dbfda34f..edcad828 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,23 +11,23 @@ SYNOPSIS :: - cdist [-h] [-d] [-v] [-V] {banner,config,shell,install} ... + cdist [-h] [-v] [-V] {banner,config,shell,install} ... - cdist banner [-h] [-d] [-v] + cdist banner [-h] [-v] - cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + cdist config [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s] [host [host ...]] - cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + cdist install [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s] [host [host ...]] - cdist shell [-h] [-d] [-v] [-s SHELL] + cdist shell [-h] [-v] [-s SHELL] DESCRIPTION @@ -46,10 +46,6 @@ All commands accept the following options: Show the help screen -.. option:: -d, --debug - - Set log level to debug (deprecated, use -vvv instead) - .. option:: -q, --quiet Quiet mode: disables logging, including WARNING and ERROR @@ -211,7 +207,7 @@ EXAMPLES .. code-block:: sh # Configure ikq05.ethz.ch with debug enabled - % cdist config -d ikq05.ethz.ch + % cdist config -vvv ikq05.ethz.ch # Configure hosts in parallel and use a different configuration directory % cdist config -c ~/p/cdist-nutzung \ @@ -245,7 +241,7 @@ EXAMPLES [--group GROUP] [--owner OWNER] [--mode MODE] object_id # Install ikq05.ethz.ch with debug enabled - % cdist install -d ikq05.ethz.ch + % cdist install -vvv ikq05.ethz.ch ENVIRONMENT ----------- diff --git a/scripts/cdist b/scripts/cdist index 605c5a6c..b55091f9 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -40,12 +40,10 @@ def commandline(): args = parser['main'].parse_args(sys.argv[1:]) # Loglevels are handled globally in here - retval = cdist.argparse.handle_loglevel(args) - if retval: - log.warning(retval) + cdist.argparse.handle_loglevel(args) - log.trace(args) log.info("version %s" % cdist.VERSION) + log.trace(args) # Work around python 3.3 bug: # http://bugs.python.org/issue16308 From f6e1174adb16d89af78f2dbfbafd1c1eda167c67 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 3 Jul 2017 13:32:39 +0200 Subject: [PATCH 11/23] start moving info log messages to verbose Signed-off-by: Steven Armstrong --- cdist/config.py | 13 ++++++------- cdist/core/explorer.py | 4 ++-- cdist/core/manifest.py | 4 ++-- scripts/cdist | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b17ec67f..d0478c8c 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -166,7 +166,7 @@ class Config(object): failed_hosts.append(host) time_end = time.time() - log.info("Total processing time for %s host(s): %s", hostcnt, + log.verbose("Total processing time for %s host(s): %s", hostcnt, (time_end - time_start)) if len(failed_hosts) > 0: @@ -466,8 +466,8 @@ class Config(object): def object_prepare(self, cdist_object): """Prepare object: Run type explorer + manifest""" - self.log.info("Preparing object {}".format(cdist_object.name)) - self.log.info( + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( "Running manifest and explorers for " + cdist_object.name) self.explorer.run_type_explorers(cdist_object) self.manifest.run_type_manifest(cdist_object) @@ -476,8 +476,7 @@ class Config(object): def object_run(self, cdist_object): """Run gencode and code for an object""" - self.log.info("Running object " + cdist_object.name) - self.log.verbose("Trying to run object %s" % (cdist_object.name)) + self.log.verbose("Running object " + cdist_object.name) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " "object: %s"), cdist_object) @@ -494,7 +493,7 @@ class Config(object): # Execute if not self.dry_run: if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Executing code for %s" % (cdist_object.name)) + self.log.info("Updating %s" % (cdist_object.name)) if cdist_object.code_local: self.log.trace("Executing local code for %s" % (cdist_object.name)) @@ -505,7 +504,7 @@ class Config(object): self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) else: - self.log.info("Skipping code execution due to DRY RUN") + self.log.verbose("Skipping code execution due to DRY RUN") # Mark this object as done self.log.trace("Finishing run of " + cdist_object.name) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index b5c00897..0d47e45e 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -95,7 +95,7 @@ class Explorer(object): out_path directory. """ - self.log.info("Running global explorers") + self.log.verbose("Running global explorers") self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) @@ -168,7 +168,7 @@ class Explorer(object): in the object. """ - self.log.info("Running type explorers for {}".format( + self.log.verbose("Running type explorers for {}".format( cdist_object.cdist_type)) self.log.trace("Transfering type explorers for type: %s", cdist_object.cdist_type) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 16c08353..d8570097 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -149,7 +149,7 @@ class Manifest(object): raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" - self.log.info("Running initial manifest " + initial_manifest) + self.log.verbose("Running initial manifest " + initial_manifest) self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix, @@ -176,7 +176,7 @@ class Manifest(object): cdist_object.cdist_type.manifest_path) message_prefix = cdist_object.name if os.path.isfile(type_manifest): - self.log.info("Running type manifest " + type_manifest) + self.log.verbose("Running type manifest " + type_manifest) self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix, diff --git a/scripts/cdist b/scripts/cdist index b55091f9..81220ca3 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -42,7 +42,7 @@ def commandline(): # Loglevels are handled globally in here cdist.argparse.handle_loglevel(args) - log.info("version %s" % cdist.VERSION) + log.verbose("version %s" % cdist.VERSION) log.trace(args) # Work around python 3.3 bug: From a1d73f72811c9675952d070b5c84e969a698a3c9 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Jul 2017 16:46:27 +0200 Subject: [PATCH 12/23] log changing object as Processing instead of Updating Signed-off-by: Steven Armstrong --- cdist/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/config.py b/cdist/config.py index d0478c8c..0d847dea 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -493,7 +493,7 @@ class Config(object): # Execute if not self.dry_run: if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Updating %s" % (cdist_object.name)) + self.log.info("Processing %s" % (cdist_object.name)) if cdist_object.code_local: self.log.trace("Executing local code for %s" % (cdist_object.name)) From 9fe242054709e2de7e149a56a1813577d4c23db8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Wed, 5 Jul 2017 16:40:02 +0200 Subject: [PATCH 13/23] also log start of config run Signed-off-by: Steven Armstrong --- cdist/config.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 0d847dea..eee47d03 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -269,6 +269,8 @@ class Config(object): """Do what is most often done: deploy & cleanup""" start_time = time.time() + self.log.info("Starting configuration run") + self._init_files_dirs() self.explorer.run_global_explorers(self.local.global_explorer_out_path) @@ -276,8 +278,8 @@ class Config(object): self.iterate_until_finished() self.local.save_cache(start_time) - self.log.info("Finished successful run in %s seconds", - time.time() - start_time) + self.log.info("Finished successful run in {:.2f} seconds".format( + time.time() - start_time)) def object_list(self): """Short name for object list retrieval""" From 93ab184e3af6e44600d043b5bbd1220c713042d0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 6 Jul 2017 07:34:08 +0200 Subject: [PATCH 14/23] Update changelog --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index 976fa600..72ae5737 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,6 +7,8 @@ next: * Core: Allow manifest and gencode scripts to be written in any language (Darko Poljak) * Documentation: Improvements to the english and fix typos (Mesar Hameed) * Core: Merge -C custom cache path pattern option from beta branch (Darko Poljak) + * Core: Improve and cleanup logging (Darko Poljak, Steven Armstrong) + * Core: Remove deprecated -d option (Darko Poljak) 4.4.4: 2017-06-16 * Core: Support -j parallelization for object prepare and object run (Darko Poljak) From 80c3dd7572329b4352b5637767b9d1786ce41011 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 6 Jul 2017 13:30:57 +0200 Subject: [PATCH 15/23] Ignore directory entries that begin with dot('.'). --- cdist/core/__init__.py | 1 + cdist/core/cdist_object.py | 2 +- cdist/core/cdist_type.py | 9 +++++---- cdist/core/util.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 cdist/core/util.py diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index 41e00a3a..8c384b3c 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -27,3 +27,4 @@ from cdist.core.cdist_object import IllegalObjectIdError from cdist.core.explorer import Explorer from cdist.core.manifest import Manifest from cdist.core.code import Code +from cdist.core.util import listdir diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 262db8bf..3167792b 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -107,7 +107,7 @@ class CdistObject(object): @classmethod def list_type_names(cls, object_base_path): """Return a list of type names""" - return os.listdir(object_base_path) + return cdist.core.listdir(object_base_path) @staticmethod def split_name(object_name): diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 14865386..9e2446f4 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -23,6 +23,7 @@ import os import cdist +import cdist.core class NoSuchTypeError(cdist.Error): @@ -75,7 +76,7 @@ class CdistType(object): @classmethod def list_type_names(cls, base_path): """Return a list of type names""" - return os.listdir(base_path) + return cdist.core.listdir(base_path) _instances = {} @@ -117,8 +118,8 @@ class CdistType(object): """Return a list of available explorers""" if not self.__explorers: try: - self.__explorers = os.listdir(os.path.join(self.absolute_path, - "explorer")) + self.__explorers = cdist.core.listdir( + os.path.join(self.absolute_path, "explorer")) except EnvironmentError: # error ignored self.__explorers = [] @@ -222,7 +223,7 @@ class CdistType(object): defaults_dir = os.path.join(self.absolute_path, "parameter", "default") - for name in os.listdir(defaults_dir): + for name in cdist.core.listdir(defaults_dir): try: with open(os.path.join(defaults_dir, name)) as fd: defaults[name] = fd.read().strip() diff --git a/cdist/core/util.py b/cdist/core/util.py new file mode 100644 index 00000000..a8216ef9 --- /dev/null +++ b/cdist/core/util.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + + +def listdir(path='.', include_dot=False): + """os.listdir but do not include entries whose names begin with a dot('.') + if include_dot is False. + """ + return [x for x in os.listdir(path) if include_dot or not _ishidden(x)] + + +def _ishidden(path): + return path[0] in ('.', b'.'[0]) From f9b632eebb90658a92b84420ef1b388d266c3151 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 6 Jul 2017 17:22:25 +0200 Subject: [PATCH 16/23] Improve listdir. --- cdist/core/util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/core/util.py b/cdist/core/util.py index a8216ef9..ca48c4c8 100644 --- a/cdist/core/util.py +++ b/cdist/core/util.py @@ -26,7 +26,10 @@ def listdir(path='.', include_dot=False): """os.listdir but do not include entries whose names begin with a dot('.') if include_dot is False. """ - return [x for x in os.listdir(path) if include_dot or not _ishidden(x)] + if include_dot: + return os.listdir(path) + else: + return [x for x in os.listdir(path) if not _ishidden(x)] def _ishidden(path): From 1f12faecca35ef783ef21f6672fc58da51002657 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 6 Jul 2017 22:58:58 +0200 Subject: [PATCH 17/23] Update changelog --- docs/changelog | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 72ae5737..77cc23fe 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,11 +9,12 @@ next: * Core: Merge -C custom cache path pattern option from beta branch (Darko Poljak) * Core: Improve and cleanup logging (Darko Poljak, Steven Armstrong) * Core: Remove deprecated -d option (Darko Poljak) + * Type __file: If no --source then create only if there is no file (Ander Punnar) 4.4.4: 2017-06-16 * Core: Support -j parallelization for object prepare and object run (Darko Poljak) * Type __install_mkfs: mkfs.vfat does not support -q (Nico Schottelius) - * Types __go_get, __daemontools*, __prometheus*: Fix missing dependencies, fix arguments(Kamila Součková) + * Types __go_get, __daemontools*, __prometheus*: Fix missing dependencies, fix arguments (Kamila Součková) 4.4.3: 2017-06-13 * Type __golang_from_vendor: Install golang from https://golang.org/dl/ (Kamila Součková) From d9a358bdf365056dc5302bf086943716574ff168 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 7 Jul 2017 02:10:06 +0200 Subject: [PATCH 18/23] add forgotten symlink Signed-off-by: Steven Armstrong --- cdist/conf/type/__install_chroot_mount/gencode-local | 1 + 1 file changed, 1 insertion(+) create mode 120000 cdist/conf/type/__install_chroot_mount/gencode-local diff --git a/cdist/conf/type/__install_chroot_mount/gencode-local b/cdist/conf/type/__install_chroot_mount/gencode-local new file mode 120000 index 00000000..68dcbd6a --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/gencode-local @@ -0,0 +1 @@ +../__chroot_mount/gencode-local \ No newline at end of file From 4e27c863ef11b651ec3157c8e4b893b338437d6c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 14 Jul 2017 14:50:39 +0200 Subject: [PATCH 19/23] Update changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 77cc23fe..d778993f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -10,6 +10,7 @@ next: * Core: Improve and cleanup logging (Darko Poljak, Steven Armstrong) * Core: Remove deprecated -d option (Darko Poljak) * Type __file: If no --source then create only if there is no file (Ander Punnar) + * Core: Ignore directory entries that begin with dot('.') (Darko Poljak) 4.4.4: 2017-06-16 * Core: Support -j parallelization for object prepare and object run (Darko Poljak) From 31899b22639d09164d5bdb15693e214415e18909 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Jul 2017 22:16:25 +0200 Subject: [PATCH 20/23] Add missing shebangs. --- cdist/conf/type/__golang_from_vendor/manifest | 2 ++ cdist/conf/type/__grafana_dashboard/manifest | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cdist/conf/type/__golang_from_vendor/manifest b/cdist/conf/type/__golang_from_vendor/manifest index 9d320830..cf164524 100755 --- a/cdist/conf/type/__golang_from_vendor/manifest +++ b/cdist/conf/type/__golang_from_vendor/manifest @@ -1 +1,3 @@ +#!/bin/sh -e + __line go_in_path --line 'export PATH=/usr/local/go/bin:$PATH' --file /etc/profile diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index b6e3020e..898770dd 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -1,3 +1,5 @@ +#!/bin/sh -e + os=$(cat $__global/explorer/os) os_version=$(cat $__global/explorer/os_version) From 126a1812a5965778461ac3f371aaba2d14db7e0b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 14 Jul 2017 21:47:30 +0200 Subject: [PATCH 21/23] Fix parallel object prepare and run steps. Add nonparallel type marker. --- cdist/conf/type/__package/nonparallel | 0 cdist/conf/type/__package_apt/nonparallel | 0 cdist/conf/type/__package_dpkg/nonparallel | 0 cdist/conf/type/__package_emerge/nonparallel | 0 .../__package_emerge_dependencies/nonparallel | 0 .../conf/type/__package_luarocks/nonparallel | 0 cdist/conf/type/__package_opkg/nonparallel | 0 cdist/conf/type/__package_pacman/nonparallel | 0 cdist/conf/type/__package_pip/nonparallel | 0 .../type/__package_pkg_freebsd/nonparallel | 0 .../type/__package_pkg_openbsd/nonparallel | 0 .../type/__package_pkgng_freebsd/nonparallel | 0 cdist/conf/type/__package_rubygem/nonparallel | 0 .../type/__package_update_index/nonparallel | 0 .../type/__package_upgrade_all/nonparallel | 0 cdist/conf/type/__package_yum/nonparallel | 0 cdist/conf/type/__package_zypper/nonparallel | 0 cdist/config.py | 87 ++++++++++++++----- cdist/core/cdist_type.py | 9 ++ cdist/core/code.py | 4 +- cdist/core/explorer.py | 13 ++- cdist/exec/remote.py | 2 +- cdist/test/cdist_type/__init__.py | 10 +++ .../fixtures/__nonparallel/nonparallel | 0 .../fixtures/__not_nonparallel/.keep | 0 25 files changed, 97 insertions(+), 28 deletions(-) create mode 100644 cdist/conf/type/__package/nonparallel create mode 100644 cdist/conf/type/__package_apt/nonparallel create mode 100644 cdist/conf/type/__package_dpkg/nonparallel create mode 100644 cdist/conf/type/__package_emerge/nonparallel create mode 100644 cdist/conf/type/__package_emerge_dependencies/nonparallel create mode 100644 cdist/conf/type/__package_luarocks/nonparallel create mode 100644 cdist/conf/type/__package_opkg/nonparallel create mode 100644 cdist/conf/type/__package_pacman/nonparallel create mode 100644 cdist/conf/type/__package_pip/nonparallel create mode 100644 cdist/conf/type/__package_pkg_freebsd/nonparallel create mode 100644 cdist/conf/type/__package_pkg_openbsd/nonparallel create mode 100644 cdist/conf/type/__package_pkgng_freebsd/nonparallel create mode 100644 cdist/conf/type/__package_rubygem/nonparallel create mode 100644 cdist/conf/type/__package_update_index/nonparallel create mode 100644 cdist/conf/type/__package_upgrade_all/nonparallel create mode 100644 cdist/conf/type/__package_yum/nonparallel create mode 100644 cdist/conf/type/__package_zypper/nonparallel create mode 100644 cdist/test/cdist_type/fixtures/__nonparallel/nonparallel create mode 100644 cdist/test/cdist_type/fixtures/__not_nonparallel/.keep diff --git a/cdist/conf/type/__package/nonparallel b/cdist/conf/type/__package/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_apt/nonparallel b/cdist/conf/type/__package_apt/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_dpkg/nonparallel b/cdist/conf/type/__package_dpkg/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_emerge/nonparallel b/cdist/conf/type/__package_emerge/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_emerge_dependencies/nonparallel b/cdist/conf/type/__package_emerge_dependencies/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_luarocks/nonparallel b/cdist/conf/type/__package_luarocks/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_opkg/nonparallel b/cdist/conf/type/__package_opkg/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pacman/nonparallel b/cdist/conf/type/__package_pacman/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pip/nonparallel b/cdist/conf/type/__package_pip/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_freebsd/nonparallel b/cdist/conf/type/__package_pkg_freebsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_openbsd/nonparallel b/cdist/conf/type/__package_pkg_openbsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkgng_freebsd/nonparallel b/cdist/conf/type/__package_pkgng_freebsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_rubygem/nonparallel b/cdist/conf/type/__package_rubygem/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_update_index/nonparallel b/cdist/conf/type/__package_update_index/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_upgrade_all/nonparallel b/cdist/conf/type/__package_upgrade_all/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_yum/nonparallel b/cdist/conf/type/__package_yum/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_zypper/nonparallel b/cdist/conf/type/__package_zypper/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/config.py b/cdist/config.py index eee47d03..fccc93a0 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -167,7 +167,7 @@ class Config(object): time_end = time.time() log.verbose("Total processing time for %s host(s): %s", hostcnt, - (time_end - time_start)) + (time_end - time_start)) if len(failed_hosts) > 0: raise cdist.Error("Failed to configure the following hosts: " + @@ -356,10 +356,34 @@ class Config(object): elif cargo: self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) + + self.log.trace("Multiprocessing cargo: %s", cargo) + + cargo_types = set() + for c in cargo: + cargo_types.add(c.cdist_type) + self.log.trace("Multiprocessing cargo_types: %s", cargo_types) + nt = len(cargo_types) + if nt == 1: + self.log.debug(("Only one type, transfering explorers " + "sequentially")) + self.explorer.transfer_type_explorers(cargo_types.pop()) + else: + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel transfering types' explorers".format( + nt))) + args = [ + (ct, ) for ct in cargo_types + ] + mp_pool_run(self.explorer.transfer_type_explorers, args, + jobs=self.jobs) + self.log.trace(("Multiprocessing for parallel transfering " + "types' explorers finished")) + self.log.trace(("Starting multiprocessing Pool for {} parallel " "objects preparation".format(n))) args = [ - (c, ) for c in cargo + (c, False, ) for c in cargo ] mp_pool_run(self.object_prepare, args, jobs=self.jobs) self.log.trace(("Multiprocessing for parallel object " @@ -382,25 +406,44 @@ class Config(object): # self.object_run(cdist_object) # objects_changed = True - cargo.append(cdist_object) - n = len(cargo) - if n == 1: - self.log.debug("Only one object, running sequentially") - self.object_run(cargo[0]) - objects_changed = True - elif cargo: - self.log.trace("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.trace(("Starting multiprocessing Pool for {} parallel " - "object run".format(n))) - args = [ - (c, ) for c in cargo - ] - mp_pool_run(self.object_run, args, jobs=self.jobs) - self.log.trace(("Multiprocessing for parallel object " - "run finished")) - objects_changed = True + # put objects in chuncks of distinct types + # so that there is no more than one object + # of the same type in one chunk because there is a + # possibility of object's process locking which + # prevents parallel execution at remote + # and do this only for nonparallel marked types + for chunk in cargo: + for obj in chunk: + if (obj.cdist_type == cdist_object.cdist_type and + cdist_object.cdist_type.is_nonparallel): + break + else: + chunk.append(cdist_object) + break + else: + chunk = [cdist_object, ] + cargo.append(chunk) + + for chunk in cargo: + self.log.trace("Running chunk: %s", chunk) + n = len(chunk) + if n == 1: + self.log.debug("Only one object, running sequentially") + self.object_run(chunk[0]) + objects_changed = True + elif chunk: + self.log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel object run".format(n))) + args = [ + (c, ) for c in chunk + ] + mp_pool_run(self.object_run, args, jobs=self.jobs) + self.log.trace(("Multiprocessing for parallel object " + "run finished")) + objects_changed = True return objects_changed @@ -466,12 +509,12 @@ class Config(object): ("The requirements of the following objects could not be " "resolved:\n%s") % ("\n".join(info_string))) - def object_prepare(self, cdist_object): + def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" self.log.verbose("Preparing object {}".format(cdist_object.name)) self.log.verbose( "Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object) + self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) self.manifest.run_type_manifest(cdist_object) cdist_object.state = core.CdistObject.STATE_PREPARED diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index c2d45cdd..1f7c3eb5 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -66,6 +66,9 @@ class CdistType(object): self.__boolean_parameters = None self.__parameter_defaults = None + def __hash__(self): + return hash(self.name) + @classmethod def list_types(cls, base_path): """Return a list of type instances""" @@ -112,6 +115,12 @@ class CdistType(object): (if not: for configuration)""" return os.path.isfile(os.path.join(self.absolute_path, "install")) + @property + def is_nonparallel(self): + """Check whether a type is a non parallel, i.e. its objects + cannot run in parallel.""" + return os.path.isfile(os.path.join(self.absolute_path, "nonparallel")) + @property def explorers(self): """Return a list of available explorers""" diff --git a/cdist/core/code.py b/cdist/core/code.py index 6917e7ba..900188bf 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -141,7 +141,9 @@ class Code(object): destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path) # FIXME: BUG: do not create destination, but top level of destination! - self.remote.mkdir(destination) + # self.remote.mkdir(destination) + # FIX? + self.remote.mkdir(os.path.dirname(destination)) self.remote.transfer(source, destination) def _run_code(self, cdist_object, which, env=None): diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 0d47e45e..d604c015 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -163,16 +163,21 @@ class Explorer(object): except EnvironmentError: return [] - def run_type_explorers(self, cdist_object): + def run_type_explorers(self, cdist_object, transfer_type_explorers=True): """Run the type explorers for the given object and save their output in the object. """ self.log.verbose("Running type explorers for {}".format( cdist_object.cdist_type)) - self.log.trace("Transfering type explorers for type: %s", - cdist_object.cdist_type) - self.transfer_type_explorers(cdist_object.cdist_type) + if transfer_type_explorers: + self.log.trace("Transfering type explorers for type: %s", + cdist_object.cdist_type) + self.transfer_type_explorers(cdist_object.cdist_type) + else: + self.log.trace(("No need for transfering type explorers for " + "type: %s"), + cdist_object.cdist_type) self.log.trace("Transfering object parameters for object: %s", cdist_object.name) self.transfer_object_parameters(cdist_object) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 2a27bae4..9588db74 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -124,7 +124,7 @@ class Remote(object): def transfer(self, source, destination, jobs=None): """Transfer a file or directory to the remote side.""" self.log.trace("Remote transfer: %s -> %s", source, destination) - self.rmdir(destination) + # self.rmdir(destination) if os.path.isdir(source): self.mkdir(destination) if jobs: diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index 6ed3f87c..6e11383b 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -113,6 +113,16 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__not_singleton') self.assertFalse(cdist_type.is_singleton) + def test_nonparallel_is_nonparallel(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__nonparallel') + self.assertTrue(cdist_type.is_nonparallel) + + def test_not_nonparallel_is_nonparallel(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__not_nonparallel') + self.assertFalse(cdist_type.is_nonparallel) + def test_install_is_install(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__install') diff --git a/cdist/test/cdist_type/fixtures/__nonparallel/nonparallel b/cdist/test/cdist_type/fixtures/__nonparallel/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__not_nonparallel/.keep b/cdist/test/cdist_type/fixtures/__not_nonparallel/.keep new file mode 100644 index 00000000..e69de29b From 660d01ce0c8f31703b7611347ecc7f5b43ed80af Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 19 Jul 2017 07:49:06 +0200 Subject: [PATCH 22/23] Update changelog --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index d778993f..fee1e1ff 100644 --- a/docs/changelog +++ b/docs/changelog @@ -11,6 +11,7 @@ next: * Core: Remove deprecated -d option (Darko Poljak) * Type __file: If no --source then create only if there is no file (Ander Punnar) * Core: Ignore directory entries that begin with dot('.') (Darko Poljak) + * Core: Fix parallel object prepare and run steps and add nonparallel type marker (Darko Poljak) 4.4.4: 2017-06-16 * Core: Support -j parallelization for object prepare and object run (Darko Poljak) From 2beb55be5417b2c11a958cc3902cc4c1e72a33f2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 19 Jul 2017 07:52:57 +0200 Subject: [PATCH 23/23] Remove FIXME comment due to fix appliance --- cdist/core/code.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 900188bf..1b5fe1d6 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -140,9 +140,6 @@ class Code(object): cdist_object.code_remote_path) destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path) - # FIXME: BUG: do not create destination, but top level of destination! - # self.remote.mkdir(destination) - # FIX? self.remote.mkdir(os.path.dirname(destination)) self.remote.transfer(source, destination)