diff --git a/cdist/__init__.py b/cdist/__init__.py index 31e59794..37651fb5 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -22,6 +22,7 @@ import os import hashlib +import cdist.log import cdist.version VERSION = cdist.version.VERSION diff --git a/cdist/argparse.py b/cdist/argparse.py index 3accedeb..8208d1f0 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -17,13 +17,19 @@ EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" parser = None +_verbosity_level_off = -2 _verbosity_level = { - 0: logging.ERROR, - 1: logging.WARNING, - 2: logging.INFO, + _verbosity_level_off: logging.OFF, + -1: logging.ERROR, + 0: logging.WARNING, + 1: logging.INFO, + 2: logging.VERBOSE, + 3: logging.DEBUG, + 4: logging.TRACE, } +# All verbosity levels above 4 are TRACE. _verbosity_level = collections.defaultdict( - lambda: logging.DEBUG, _verbosity_level) + lambda: logging.TRACE, _verbosity_level) def add_beta_command(cmd): @@ -80,16 +86,17 @@ def get_parsers(): # Options _all_ parsers have in common parser['loglevel'] = argparse.ArgumentParser(add_help=False) parser['loglevel'].add_argument( - '-d', '--debug', - help=('Set log level to debug (deprecated, use -vvv instead)'), + '-q', '--quiet', + help='Quiet mode: disables logging, including WARNING and ERROR', action='store_true', default=False) parser['loglevel'].add_argument( '-v', '--verbose', help=('Increase the verbosity level. Every instance of -v ' 'increments the verbosity level by one. Its default value ' - 'is 0. There are 4 levels of verbosity. The order of levels ' - 'from the lowest to the highest are: ERROR (0), ' - 'WARNING (1), INFO (2) and DEBUG (3 or higher).'), + 'is 0 which includes ERROR and WARNING levels. ' + 'The levels, in order from the lowest to the highest, are: ' + 'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) ' + 'TRACE (4 or higher).'), action='count', default=0) parser['beta'] = argparse.ArgumentParser(add_help=False) @@ -211,12 +218,7 @@ def get_parsers(): def handle_loglevel(args): - if args.debug: - retval = "-d/--debug is deprecated, use -vvv instead" - args.verbose = 3 - else: - retval = None + if args.quiet: + args.verbose = _verbosity_level_off logging.root.setLevel(_verbosity_level[args.verbose]) - - return retval diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index c79fc16a..aec4e43e 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -29,8 +29,11 @@ upload_file= create_file= if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then if [ ! -f "$__object/parameter/source" ]; then - create_file=1 - echo create >> "$__messages_out" + remote_stat="$(cat "$__object/explorer/stat")" + if [ -z "$remote_stat" ]; then + create_file=1 + echo create >> "$__messages_out" + fi else source="$(cat "$__object/parameter/source")" if [ "$source" = "-" ]; then diff --git a/cdist/conf/type/__golang_from_vendor/manifest b/cdist/conf/type/__golang_from_vendor/manifest index 9d320830..cf164524 100755 --- a/cdist/conf/type/__golang_from_vendor/manifest +++ b/cdist/conf/type/__golang_from_vendor/manifest @@ -1 +1,3 @@ +#!/bin/sh -e + __line go_in_path --line 'export PATH=/usr/local/go/bin:$PATH' --file /etc/profile diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest index b6e3020e..898770dd 100755 --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -1,3 +1,5 @@ +#!/bin/sh -e + os=$(cat $__global/explorer/os) os_version=$(cat $__global/explorer/os_version) diff --git a/cdist/conf/type/__install_chroot_mount/gencode-local b/cdist/conf/type/__install_chroot_mount/gencode-local new file mode 120000 index 00000000..68dcbd6a --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/gencode-local @@ -0,0 +1 @@ +../__chroot_mount/gencode-local \ No newline at end of file diff --git a/cdist/conf/type/__package/nonparallel b/cdist/conf/type/__package/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_apt/nonparallel b/cdist/conf/type/__package_apt/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_dpkg/nonparallel b/cdist/conf/type/__package_dpkg/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_emerge/nonparallel b/cdist/conf/type/__package_emerge/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_emerge_dependencies/nonparallel b/cdist/conf/type/__package_emerge_dependencies/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_luarocks/nonparallel b/cdist/conf/type/__package_luarocks/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_opkg/nonparallel b/cdist/conf/type/__package_opkg/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pacman/nonparallel b/cdist/conf/type/__package_pacman/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pip/nonparallel b/cdist/conf/type/__package_pip/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_freebsd/nonparallel b/cdist/conf/type/__package_pkg_freebsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_openbsd/nonparallel b/cdist/conf/type/__package_pkg_openbsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkgng_freebsd/nonparallel b/cdist/conf/type/__package_pkgng_freebsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_rubygem/nonparallel b/cdist/conf/type/__package_rubygem/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_update_index/nonparallel b/cdist/conf/type/__package_update_index/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_upgrade_all/nonparallel b/cdist/conf/type/__package_upgrade_all/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_yum/nonparallel b/cdist/conf/type/__package_yum/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_zypper/nonparallel b/cdist/conf/type/__package_zypper/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/config.py b/cdist/config.py index bd9d7bf8..48284e8e 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -151,7 +151,7 @@ class Config(object): hostcnt += 1 if args.parallel: - log.debug("Creating child process for %s", host) + log.trace("Creating child process for %s", host) process[host] = multiprocessing.Process( target=cls.onehost, args=(host, host_base_path, hostdir, args, True)) @@ -166,15 +166,15 @@ class Config(object): # Catch errors in parallel mode when joining if args.parallel: for host in process.keys(): - log.debug("Joining process %s", host) + log.trace("Joining process %s", host) process[host].join() if not process[host].exitcode == 0: failed_hosts.append(host) time_end = time.time() - log.info("Total processing time for %s host(s): %s", hostcnt, - (time_end - time_start)) + log.verbose("Total processing time for %s host(s): %s", hostcnt, + (time_end - time_start)) if len(failed_hosts) > 0: raise cdist.Error("Failed to configure the following hosts: " + @@ -233,13 +233,15 @@ class Config(object): host_dir_name=host_dir_name, initial_manifest=args.manifest, add_conf_dirs=args.conf_dir, - cache_path_pattern=args.cache_path_pattern) + cache_path_pattern=args.cache_path_pattern, + quiet_mode=args.quiet) remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, remote_copy=remote_copy, - base_path=args.remote_out_path) + base_path=args.remote_out_path, + quiet_mode=args.quiet) cleanup_cmds = [] if cleanup_cmd: @@ -284,6 +286,8 @@ class Config(object): """Do what is most often done: deploy & cleanup""" start_time = time.time() + self.log.info("Starting configuration run") + self._init_files_dirs() self.explorer.run_global_explorers(self.local.global_explorer_out_path) @@ -292,8 +296,8 @@ class Config(object): self.cleanup() self.local.save_cache(start_time) - self.log.info("Finished successful run in %s seconds", - time.time() - start_time) + self.log.info("Finished successful run in {:.2f} seconds".format( + time.time() - start_time)) def cleanup(self): self.log.debug("Running cleanup commands") @@ -329,7 +333,7 @@ class Config(object): return objects_changed def _iterate_once_sequential(self): - self.log.info("Iteration in sequential mode") + self.log.debug("Iteration in sequential mode") objects_changed = False for cdist_object in self.object_list(): @@ -356,7 +360,7 @@ class Config(object): return objects_changed def _iterate_once_parallel(self): - self.log.info("Iteration in parallel mode in {} jobs".format( + self.log.debug("Iteration in parallel mode in {} jobs".format( self.jobs)) objects_changed = False @@ -379,15 +383,39 @@ class Config(object): self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for {} parallel " + + self.log.trace("Multiprocessing cargo: %s", cargo) + + cargo_types = set() + for c in cargo: + cargo_types.add(c.cdist_type) + self.log.trace("Multiprocessing cargo_types: %s", cargo_types) + nt = len(cargo_types) + if nt == 1: + self.log.debug(("Only one type, transfering explorers " + "sequentially")) + self.explorer.transfer_type_explorers(cargo_types.pop()) + else: + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel transfering types' explorers".format( + nt))) + args = [ + (ct, ) for ct in cargo_types + ] + mp_pool_run(self.explorer.transfer_type_explorers, args, + jobs=self.jobs) + self.log.trace(("Multiprocessing for parallel transfering " + "types' explorers finished")) + + self.log.trace(("Starting multiprocessing Pool for {} parallel " "objects preparation".format(n))) args = [ - (c, ) for c in cargo + (c, False, ) for c in cargo ] mp_pool_run(self.object_prepare, args, jobs=self.jobs) - self.log.debug(("Multiprocessing for parallel object " + self.log.trace(("Multiprocessing for parallel object " "preparation finished")) objects_changed = True @@ -407,25 +435,44 @@ class Config(object): # self.object_run(cdist_object) # objects_changed = True - cargo.append(cdist_object) - n = len(cargo) - if n == 1: - self.log.debug("Only one object, running sequentially") - self.object_run(cargo[0]) - objects_changed = True - elif cargo: - self.log.debug("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for {} parallel " - "object run".format(n))) - args = [ - (c, ) for c in cargo - ] - mp_pool_run(self.object_run, args, jobs=self.jobs) - self.log.debug(("Multiprocessing for parallel object " - "run finished")) - objects_changed = True + # put objects in chuncks of distinct types + # so that there is no more than one object + # of the same type in one chunk because there is a + # possibility of object's process locking which + # prevents parallel execution at remote + # and do this only for nonparallel marked types + for chunk in cargo: + for obj in chunk: + if (obj.cdist_type == cdist_object.cdist_type and + cdist_object.cdist_type.is_nonparallel): + break + else: + chunk.append(cdist_object) + break + else: + chunk = [cdist_object, ] + cargo.append(chunk) + + for chunk in cargo: + self.log.trace("Running chunk: %s", chunk) + n = len(chunk) + if n == 1: + self.log.debug("Only one object, running sequentially") + self.object_run(chunk[0]) + objects_changed = True + elif chunk: + self.log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel object run".format(n))) + args = [ + (c, ) for c in chunk + ] + mp_pool_run(self.object_run, args, jobs=self.jobs) + self.log.trace(("Multiprocessing for parallel object " + "run finished")) + objects_changed = True return objects_changed @@ -491,18 +538,19 @@ class Config(object): ("The requirements of the following objects could not be " "resolved:\n%s") % ("\n".join(info_string))) - def object_prepare(self, cdist_object): + def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" - self.log.info( + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( "Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object) + self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) self.manifest.run_type_manifest(cdist_object) cdist_object.state = core.CdistObject.STATE_PREPARED def object_run(self, cdist_object): """Run gencode and code for an object""" - self.log.debug("Trying to run object %s" % (cdist_object.name)) + self.log.verbose("Running object " + cdist_object.name) if cdist_object.state == core.CdistObject.STATE_DONE: raise cdist.Error(("Attempting to run an already finished " "object: %s"), cdist_object) @@ -510,7 +558,7 @@ class Config(object): cdist_type = cdist_object.cdist_type # Generate - self.log.info("Generating code for %s" % (cdist_object.name)) + self.log.debug("Generating code for %s" % (cdist_object.name)) cdist_object.code_local = self.code.run_gencode_local(cdist_object) cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) if cdist_object.code_local or cdist_object.code_remote: @@ -519,15 +567,19 @@ class Config(object): # Execute if not self.dry_run: if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Executing code for %s" % (cdist_object.name)) + self.log.info("Processing %s" % (cdist_object.name)) if cdist_object.code_local: + self.log.trace("Executing local code for %s" + % (cdist_object.name)) self.code.run_code_local(cdist_object) if cdist_object.code_remote: + self.log.trace("Executing remote code for %s" + % (cdist_object.name)) self.code.transfer_code_remote(cdist_object) self.code.run_code_remote(cdist_object) else: - self.log.info("Skipping code execution due to DRY RUN") + self.log.verbose("Skipping code execution due to DRY RUN") # Mark this object as done - self.log.debug("Finishing run of " + cdist_object.name) + self.log.trace("Finishing run of " + cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index 41e00a3a..8c384b3c 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -27,3 +27,4 @@ from cdist.core.cdist_object import IllegalObjectIdError from cdist.core.explorer import Explorer from cdist.core.manifest import Manifest from cdist.core.code import Code +from cdist.core.util import listdir diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 262db8bf..2d92aa41 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -22,7 +22,6 @@ # import fnmatch -import logging import os import collections @@ -30,8 +29,6 @@ import cdist import cdist.core from cdist.util import fsproperty -log = logging.getLogger(__name__) - class IllegalObjectIdError(cdist.Error): def __init__(self, object_id, message=None): @@ -107,7 +104,7 @@ class CdistObject(object): @classmethod def list_type_names(cls, object_base_path): """Return a list of type names""" - return os.listdir(object_base_path) + return cdist.core.listdir(object_base_path) @staticmethod def split_name(object_name): diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 14865386..1f7c3eb5 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -21,8 +21,8 @@ # import os - import cdist +import cdist.core class NoSuchTypeError(cdist.Error): @@ -66,6 +66,9 @@ class CdistType(object): self.__boolean_parameters = None self.__parameter_defaults = None + def __hash__(self): + return hash(self.name) + @classmethod def list_types(cls, base_path): """Return a list of type instances""" @@ -75,7 +78,7 @@ class CdistType(object): @classmethod def list_type_names(cls, base_path): """Return a list of type names""" - return os.listdir(base_path) + return cdist.core.listdir(base_path) _instances = {} @@ -112,13 +115,19 @@ class CdistType(object): (if not: for configuration)""" return os.path.isfile(os.path.join(self.absolute_path, "install")) + @property + def is_nonparallel(self): + """Check whether a type is a non parallel, i.e. its objects + cannot run in parallel.""" + return os.path.isfile(os.path.join(self.absolute_path, "nonparallel")) + @property def explorers(self): """Return a list of available explorers""" if not self.__explorers: try: - self.__explorers = os.listdir(os.path.join(self.absolute_path, - "explorer")) + self.__explorers = cdist.core.listdir( + os.path.join(self.absolute_path, "explorer")) except EnvironmentError: # error ignored self.__explorers = [] @@ -222,7 +231,7 @@ class CdistType(object): defaults_dir = os.path.join(self.absolute_path, "parameter", "default") - for name in os.listdir(defaults_dir): + for name in cdist.core.listdir(defaults_dir): try: with open(os.path.join(defaults_dir, name)) as fd: defaults[name] = fd.read().strip() diff --git a/cdist/core/code.py b/cdist/core/code.py index e9e2edf0..1b5fe1d6 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -21,13 +21,10 @@ # # -import logging import os import cdist -log = logging.getLogger(__name__) - ''' common: @@ -143,8 +140,7 @@ class Code(object): cdist_object.code_remote_path) destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path) - # FIXME: BUG: do not create destination, but top level of destination! - self.remote.mkdir(destination) + self.remote.mkdir(os.path.dirname(destination)) self.remote.transfer(source, destination) def _run_code(self, cdist_object, which, env=None): diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 45afc5c0..d604c015 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -95,7 +95,7 @@ class Explorer(object): out_path directory. """ - self.log.info("Running global explorers") + self.log.verbose("Running global explorers") self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) @@ -109,22 +109,22 @@ class Explorer(object): fd.write(output) def _run_global_explorers_seq(self, out_path): - self.log.info("Running global explorers sequentially") + self.log.debug("Running global explorers sequentially") for explorer in self.list_global_explorer_names(): self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.info("Running global explorers in {} parallel jobs".format( + self.log.debug("Running global explorers in {} parallel jobs".format( self.jobs)) - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for global " + self.log.trace(("Starting multiprocessing Pool for global " "explorers run")) args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.debug(("Multiprocessing run for global explorers " + self.log.trace(("Multiprocessing run for global explorers " "finished")) # logger is not pickable, so remove it when we pickle @@ -163,20 +163,27 @@ class Explorer(object): except EnvironmentError: return [] - def run_type_explorers(self, cdist_object): + def run_type_explorers(self, cdist_object, transfer_type_explorers=True): """Run the type explorers for the given object and save their output in the object. """ - self.log.debug("Transfering type explorers for type: %s", - cdist_object.cdist_type) - self.transfer_type_explorers(cdist_object.cdist_type) - self.log.debug("Transfering object parameters for object: %s", + self.log.verbose("Running type explorers for {}".format( + cdist_object.cdist_type)) + if transfer_type_explorers: + self.log.trace("Transfering type explorers for type: %s", + cdist_object.cdist_type) + self.transfer_type_explorers(cdist_object.cdist_type) + else: + self.log.trace(("No need for transfering type explorers for " + "type: %s"), + cdist_object.cdist_type) + self.log.trace("Transfering object parameters for object: %s", cdist_object.name) self.transfer_object_parameters(cdist_object) for explorer in self.list_type_explorer_names(cdist_object.cdist_type): output = self.run_type_explorer(explorer, cdist_object) - self.log.debug("Running type explorer '%s' for object '%s'", + self.log.trace("Running type explorer '%s' for object '%s'", explorer, cdist_object.name) cdist_object.explorers[explorer] = output @@ -203,7 +210,7 @@ class Explorer(object): remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.debug("Skipping retransfer of type explorers for: %s", + self.log.trace("Skipping retransfer of type explorers for: %s", cdist_type) else: source = os.path.join(self.local.type_path, diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 29f96c4f..d8570097 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -145,12 +145,11 @@ class Manifest(object): else: user_supplied = True - self.log.info("Running initial manifest " + initial_manifest) - if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" + self.log.verbose("Running initial manifest " + initial_manifest) self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix, @@ -177,6 +176,7 @@ class Manifest(object): cdist_object.cdist_type.manifest_path) message_prefix = cdist_object.name if os.path.isfile(type_manifest): + self.log.verbose("Running type manifest " + type_manifest) self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix, diff --git a/cdist/core/util.py b/cdist/core/util.py new file mode 100644 index 00000000..ca48c4c8 --- /dev/null +++ b/cdist/core/util.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + + +def listdir(path='.', include_dot=False): + """os.listdir but do not include entries whose names begin with a dot('.') + if include_dot is False. + """ + if include_dot: + return os.listdir(path) + else: + return [x for x in os.listdir(path) if not _ishidden(x)] + + +def _ishidden(path): + return path[0] in ('.', b'.'[0]) diff --git a/cdist/emulator.py b/cdist/emulator.py index cdbe5b08..7c9dfcca 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -98,7 +98,7 @@ class Emulator(object): self.save_stdin() self.record_requirements() self.record_auto_requirements() - self.log.debug("Finished %s %s" % ( + self.log.trace("Finished %s %s" % ( self.cdist_object.path, self.parameters)) def __init_log(self): @@ -148,7 +148,7 @@ class Emulator(object): # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.debug('Args: %s' % self.args) + self.log.trace('Args: %s' % self.args) def setup_object(self): # Setup object - and ensure it is not in args @@ -256,10 +256,10 @@ class Emulator(object): # (this would leed to an circular dependency) if ("CDIST_ORDER_DEPENDENCY" in self.env and 'CDIST_OVERRIDE' not in self.env): - # load object name created bevor this one from typeorder file ... + # load object name created befor this one from typeorder file ... with open(self.typeorder_path, 'r') as typecreationfile: typecreationorder = typecreationfile.readlines() - # get the type created bevore this one ... + # get the type created before this one ... try: lastcreatedtype = typecreationorder[-2].strip() if 'require' in self.env: diff --git a/cdist/exec/local.py b/cdist/exec/local.py index fc2e7c09..6c285204 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -54,7 +54,8 @@ class Local(object): exec_path=sys.argv[0], initial_manifest=None, add_conf_dirs=None, - cache_path_pattern=None): + cache_path_pattern=None, + quiet_mode=False): self.target_host = target_host self.hostdir = host_dir_name @@ -64,6 +65,7 @@ class Local(object): self.custom_initial_manifest = initial_manifest self._add_conf_dirs = add_conf_dirs self.cache_path_pattern = cache_path_pattern + self.quiet_mode = quiet_mode self._init_log() self._init_permissions() @@ -163,7 +165,7 @@ class Local(object): with open(self.object_marker_file, 'w') as fd: fd.write("%s\n" % self.object_marker_name) - self.log.debug("Object marker %s saved in %s" % ( + self.log.trace("Object marker %s saved in %s" % ( self.object_marker_name, self.object_marker_file)) def _init_cache_dir(self, cache_dir): @@ -178,12 +180,12 @@ class Local(object): def rmdir(self, path): """Remove directory on the local side.""" - self.log.debug("Local rmdir: %s", path) + self.log.trace("Local rmdir: %s", path) shutil.rmtree(path) def mkdir(self, path): """Create directory on the local side.""" - self.log.debug("Local mkdir: %s", path) + self.log.trace("Local mkdir: %s", path) os.makedirs(path, exist_ok=True) def run(self, command, env=None, return_output=False, message_prefix=None, @@ -192,7 +194,6 @@ class Local(object): Return the output as a string. """ - self.log.debug("Local run: %s", command) assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: %s" % command) @@ -211,19 +212,30 @@ class Local(object): message = cdist.message.Message(message_prefix, self.messages_path) env.update(message.env) + self.log.trace("Local run: %s", command) try: + if self.quiet_mode: + stderr = subprocess.DEVNULL + else: + stderr = None if save_output: - output, errout = exec_util.call_get_output(command, env=env) - self.log.debug("Local stdout: {}".format(output)) + output, errout = exec_util.call_get_output( + command, env=env, stderr=stderr) + self.log.trace("Local stdout: {}".format(output)) # Currently, stderr is not captured. - # self.log.debug("Local stderr: {}".format(errout)) + # self.log.trace("Local stderr: {}".format(errout)) if return_output: return output.decode() else: # In some cases no output is saved. # This is used for shell command, stdout and stderr # must not be catched. - subprocess.check_call(command, env=env) + if self.quiet_mode: + stdout = subprocess.DEVNULL + else: + stdout = None + subprocess.check_call(command, env=env, stderr=stderr, + stdout=stdout) except subprocess.CalledProcessError as e: exec_util.handle_called_process_error(e, command) except OSError as error: @@ -279,13 +291,14 @@ class Local(object): return cache_subpath def save_cache(self, start_time=time.time()): - self.log.debug("cache subpath pattern: {}".format( + self.log.trace("cache subpath pattern: {}".format( self.cache_path_pattern)) cache_subpath = self._cache_subpath(start_time, self.cache_path_pattern) self.log.debug("cache subpath: {}".format(cache_subpath)) destination = os.path.join(self.cache_path, cache_subpath) - self.log.debug("Saving " + self.base_path + " to " + destination) + self.log.trace(("Saving cache: " + self.base_path + " to " + + destination)) if not os.path.exists(destination): shutil.move(self.base_path, destination) @@ -340,7 +353,7 @@ class Local(object): if os.path.exists(dst): os.unlink(dst) - self.log.debug("Linking %s to %s ..." % (src, dst)) + self.log.trace("Linking %s to %s ..." % (src, dst)) try: os.symlink(src, dst) except OSError as e: @@ -352,7 +365,7 @@ class Local(object): src = os.path.abspath(self.exec_path) for cdist_type in core.CdistType.list_types(self.type_path): dst = os.path.join(self.bin_path, cdist_type.name) - self.log.debug("Linking emulator: %s to %s", src, dst) + self.log.trace("Linking emulator: %s to %s", src, dst) try: os.symlink(src, dst) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 042b7103..9588db74 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -62,7 +62,8 @@ class Remote(object): target_host, remote_exec, remote_copy, - base_path=None): + base_path=None, + quiet_mode=None): self.target_host = target_host self._exec = remote_exec self._copy = remote_copy @@ -71,6 +72,7 @@ class Remote(object): self.base_path = base_path else: self.base_path = "/var/lib/cdist" + self.quiet_mode = quiet_mode self.conf_path = os.path.join(self.base_path, "conf") self.object_path = os.path.join(self.base_path, "object") @@ -111,18 +113,18 @@ class Remote(object): def rmdir(self, path): """Remove directory on the remote side.""" - self.log.debug("Remote rmdir: %s", path) + self.log.trace("Remote rmdir: %s", path) self.run(["rm", "-rf", path]) def mkdir(self, path): """Create directory on the remote side.""" - self.log.debug("Remote mkdir: %s", path) + self.log.trace("Remote mkdir: %s", path) self.run(["mkdir", "-p", path]) def transfer(self, source, destination, jobs=None): """Transfer a file or directory to the remote side.""" - self.log.debug("Remote transfer: %s -> %s", source, destination) - self.rmdir(destination) + self.log.trace("Remote transfer: %s -> %s", source, destination) + # self.rmdir(destination) if os.path.isdir(source): self.mkdir(destination) if jobs: @@ -147,11 +149,11 @@ class Remote(object): def _transfer_dir_parallel(self, source, destination, jobs): """Transfer a directory to the remote side in parallel mode.""" - self.log.info("Remote transfer in {} parallel jobs".format( + self.log.debug("Remote transfer in {} parallel jobs".format( jobs)) - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for parallel " + self.log.trace(("Starting multiprocessing Pool for parallel " "remote transfer")) args = [] for f in glob.glob1(source, '*'): @@ -161,7 +163,7 @@ class Remote(object): _wrap_addr(self.target_host[0]), destination)]) args.append((command, )) mp_pool_run(self._run_command, args, jobs=jobs) - self.log.debug(("Multiprocessing for parallel transfer " + self.log.trace(("Multiprocessing for parallel transfer " "finished")) def run_script(self, script, env=None, return_output=False): @@ -226,12 +228,17 @@ class Remote(object): os_environ['__target_hostname'] = self.target_host[1] os_environ['__target_fqdn'] = self.target_host[2] - self.log.debug("Remote run: %s", command) + self.log.trace("Remote run: %s", command) try: - output, errout = exec_util.call_get_output(command, env=os_environ) - self.log.debug("Remote stdout: {}".format(output)) + if self.quiet_mode: + stderr = subprocess.DEVNULL + else: + stderr = None + output, errout = exec_util.call_get_output( + command, env=os_environ, stderr=stderr) + self.log.trace("Remote stdout: {}".format(output)) # Currently, stderr is not captured. - # self.log.debug("Remote stderr: {}".format(errout)) + # self.log.trace("Remote stderr: {}".format(errout)) if return_output: return output.decode() except subprocess.CalledProcessError as e: diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 864a73a3..9ed7103b 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -116,14 +116,14 @@ import cdist # return (result.stdout, result.stderr) -def call_get_output(command, env=None): +def call_get_output(command, env=None, stderr=None): """Run the given command with the given environment. Return the tuple of stdout and stderr output as a byte strings. """ assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: {}".format(command)) - return (_call_get_stdout(command, env), None) + return (_call_get_stdout(command, env, stderr), None) def handle_called_process_error(err, command): @@ -140,7 +140,7 @@ def handle_called_process_error(err, command): err.returncode, err.output)) -def _call_get_stdout(command, env=None): +def _call_get_stdout(command, env=None, stderr=None): """Run the given command with the given environment. Return the stdout output as a byte string, stderr is ignored. """ @@ -148,7 +148,7 @@ def _call_get_stdout(command, env=None): "list or tuple argument expected, got: {}".format(command)) with TemporaryFile() as fout: - subprocess.check_call(command, env=env, stdout=fout) + subprocess.check_call(command, env=env, stdout=fout, stderr=stderr) fout.seek(0) output = fout.read() diff --git a/cdist/log.py b/cdist/log.py index 2341c282..ce0addcc 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -23,6 +23,31 @@ import logging +# Define additional cdist logging levels. +logging.OFF = logging.CRITICAL + 10 # disable logging +logging.addLevelName(logging.OFF, 'OFF') + +logging.VERBOSE = logging.INFO - 5 +logging.addLevelName(logging.VERBOSE, 'VERBOSE') + + +def _verbose(msg, *args, **kwargs): + logging.log(logging.VERBOSE, msg, *args, **kwargs) + + +logging.verbose = _verbose + +logging.TRACE = logging.DEBUG - 5 +logging.addLevelName(logging.TRACE, 'TRACE') + + +def _trace(msg, *args, **kwargs): + logging.log(logging.TRACE, msg, *args, **kwargs) + + +logging.trace = _trace + + class Log(logging.Logger): def __init__(self, name): @@ -37,3 +62,13 @@ class Log(logging.Logger): record.msg = self.name + ": " + str(record.msg) return True + + def verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, *args, **kwargs) + + def trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) + + +logging.setLoggerClass(Log) +logging.basicConfig(format='%(levelname)s: %(message)s') diff --git a/cdist/shell.py b/cdist/shell.py index 9378efc3..662f8f7d 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -86,10 +86,10 @@ class Shell(object): self._init_files_dirs() self._init_environment() - log.info("Starting shell...") + log.trace("Starting shell...") # save_output=False -> do not catch stdout and stderr self.local.run([self.shell], self.env, save_output=False) - log.info("Finished shell.") + log.trace("Finished shell.") @classmethod def commandline(cls, args): diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index 6ed3f87c..6e11383b 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -113,6 +113,16 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__not_singleton') self.assertFalse(cdist_type.is_singleton) + def test_nonparallel_is_nonparallel(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__nonparallel') + self.assertTrue(cdist_type.is_nonparallel) + + def test_not_nonparallel_is_nonparallel(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__not_nonparallel') + self.assertFalse(cdist_type.is_nonparallel) + def test_install_is_install(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__install') diff --git a/cdist/test/cdist_type/fixtures/__nonparallel/nonparallel b/cdist/test/cdist_type/fixtures/__nonparallel/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__not_nonparallel/.keep b/cdist/test/cdist_type/fixtures/__not_nonparallel/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index a747aeb5..0bcb0a83 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -45,8 +45,8 @@ def resolve_target_host_name(host): log.debug("derived host_name for host \"{}\": {}".format( host, host_name)) except (socket.gaierror, socket.herror) as e: - log.warn("Could not derive host_name for {}" - ", $host_name will be empty. Error is: {}".format(host, e)) + log.warning("Could not derive host_name for {}" + ", $host_name will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_name = '' return host_name @@ -59,8 +59,8 @@ def resolve_target_fqdn(host): log.debug("derived host_fqdn for host \"{}\": {}".format( host, host_fqdn)) except socket.herror as e: - log.warn("Could not derive host_fqdn for {}" - ", $host_fqdn will be empty. Error is: {}".format(host, e)) + log.warning("Could not derive host_fqdn for {}" + ", $host_fqdn will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_fqdn = '' return host_fqdn diff --git a/docs/changelog b/docs/changelog index 976fa600..fee1e1ff 100644 --- a/docs/changelog +++ b/docs/changelog @@ -7,11 +7,16 @@ next: * Core: Allow manifest and gencode scripts to be written in any language (Darko Poljak) * Documentation: Improvements to the english and fix typos (Mesar Hameed) * Core: Merge -C custom cache path pattern option from beta branch (Darko Poljak) + * Core: Improve and cleanup logging (Darko Poljak, Steven Armstrong) + * Core: Remove deprecated -d option (Darko Poljak) + * Type __file: If no --source then create only if there is no file (Ander Punnar) + * Core: Ignore directory entries that begin with dot('.') (Darko Poljak) + * Core: Fix parallel object prepare and run steps and add nonparallel type marker (Darko Poljak) 4.4.4: 2017-06-16 * Core: Support -j parallelization for object prepare and object run (Darko Poljak) * Type __install_mkfs: mkfs.vfat does not support -q (Nico Schottelius) - * Types __go_get, __daemontools*, __prometheus*: Fix missing dependencies, fix arguments(Kamila Součková) + * Types __go_get, __daemontools*, __prometheus*: Fix missing dependencies, fix arguments (Kamila Součková) 4.4.3: 2017-06-13 * Type __golang_from_vendor: Install golang from https://golang.org/dl/ (Kamila Součková) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 3bddbd02..edcad828 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,23 +11,23 @@ SYNOPSIS :: - cdist [-h] [-d] [-v] [-V] {banner,config,shell,install} ... + cdist [-h] [-v] [-V] {banner,config,shell,install} ... - cdist banner [-h] [-d] [-v] + cdist banner [-h] [-v] - cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + cdist config [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s] [host [host ...]] - cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + cdist install [-h] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [-f HOSTFILE] [-p] [-r REMOTE_OUT_PATH] [-s] [host [host ...]] - cdist shell [-h] [-d] [-v] [-s SHELL] + cdist shell [-h] [-v] [-s SHELL] DESCRIPTION @@ -46,16 +46,16 @@ All commands accept the following options: Show the help screen -.. option:: -d, --debug +.. option:: -q, --quiet - Set log level to debug (deprecated, use -vvv instead) + Quiet mode: disables logging, including WARNING and ERROR .. option:: -v, --verbose Increase the verbosity level. Every instance of -v increments the verbosity - level by one. Its default value is 0. There are 4 levels of verbosity. The - order of levels from the lowest to the highest are: ERROR (0), WARNING (1), - INFO (2) and DEBUG (3 or higher). + level by one. Its default value is 0 which includes ERROR and WARNING levels. + The levels, in order from the lowest to the highest, are: + ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4 or higher). .. option:: -V, --version @@ -207,7 +207,7 @@ EXAMPLES .. code-block:: sh # Configure ikq05.ethz.ch with debug enabled - % cdist config -d ikq05.ethz.ch + % cdist config -vvv ikq05.ethz.ch # Configure hosts in parallel and use a different configuration directory % cdist config -c ~/p/cdist-nutzung \ @@ -241,7 +241,7 @@ EXAMPLES [--group GROUP] [--owner OWNER] [--mode MODE] object_id # Install ikq05.ethz.ch with debug enabled - % cdist install -d ikq05.ethz.ch + % cdist install -vvv ikq05.ethz.ch ENVIRONMENT ----------- diff --git a/scripts/cdist b/scripts/cdist index 498091b8..81220ca3 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -40,12 +40,10 @@ def commandline(): args = parser['main'].parse_args(sys.argv[1:]) # Loglevels are handled globally in here - retval = cdist.argparse.handle_loglevel(args) - if retval: - log.warning(retval) + cdist.argparse.handle_loglevel(args) - log.debug(args) - log.info("version %s" % cdist.VERSION) + log.verbose("version %s" % cdist.VERSION) + log.trace(args) # Work around python 3.3 bug: # http://bugs.python.org/issue16308 @@ -80,10 +78,7 @@ if __name__ == "__main__": import os import re import cdist - import cdist.log - logging.setLoggerClass(cdist.log.Log) - logging.basicConfig(format='%(levelname)s: %(message)s') log = logging.getLogger("cdist") if re.match("__", os.path.basename(sys.argv[0])):