From 2a9bd77550078af2ce5bd3ba497e87c6490bbc2e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 1 Jul 2017 23:59:51 +0200 Subject: [PATCH] Merge custom cache path pattern from beta branch. --- cdist/__init__.py | 7 +++ cdist/argparse.py | 7 +++ cdist/config.py | 6 +- cdist/exec/local.py | 77 ++++++++++++++++++++------ cdist/hostsource.py | 38 +++++++------ cdist/test/exec/local.py | 37 +++++++++++++ completions/bash/cdist-completion.bash | 2 +- completions/zsh/_cdist | 2 +- docs/src/cdist-reference.rst.sh | 3 + docs/src/man1/cdist.rst | 40 +++++++++++-- 10 files changed, 176 insertions(+), 43 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index 6ea02d41..b0fd75ea 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -114,3 +114,10 @@ def str_hash(s): return hashlib.md5(s.encode('utf-8')).hexdigest() else: raise Error("Param should be string") + + +def home_dir(): + if 'HOME' in os.environ: + return os.path.join(os.environ['HOME'], ".cdist") + else: + return None diff --git a/cdist/argparse.py b/cdist/argparse.py index ea0fd159..3accedeb 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -116,6 +116,13 @@ def get_parsers(): # Config parser['config_main'] = argparse.ArgumentParser(add_help=False) + parser['config_main'].add_argument( + '-C', '--cache-path-pattern', + help=('Specify custom cache path pattern. It can also be set ' + 'by CDIST_CACHE_PATH_PATTERN environment variable. If ' + 'it is not set then default hostdir is used.'), + dest='cache_path_pattern', + default=os.environ.get('CDIST_CACHE_PATH_PATTERN')) parser['config_main'].add_argument( '-c', '--conf-dir', help=('Add configuration directory (can be repeated, ' diff --git a/cdist/config.py b/cdist/config.py index 271aea38..d342c657 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- # # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -223,7 +224,8 @@ class Config(object): base_root_path=host_base_path, host_dir_name=host_dir_name, initial_manifest=args.manifest, - add_conf_dirs=args.conf_dir) + add_conf_dirs=args.conf_dir, + cache_path_pattern=args.cache_path_pattern) remote = cdist.exec.remote.Remote( target_host=target_host, @@ -260,7 +262,7 @@ class Config(object): self.manifest.run_initial_manifest(self.local.initial_manifest) self.iterate_until_finished() - self.local.save_cache() + self.local.save_cache(start_time) self.log.info("Finished successful run in %s seconds", time.time() - start_time) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index d30bc146..fc2e7c09 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -2,7 +2,7 @@ # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -29,6 +29,8 @@ import subprocess import shutil import logging import tempfile +import time +import datetime import cdist import cdist.message @@ -51,7 +53,8 @@ class Local(object): host_dir_name, exec_path=sys.argv[0], initial_manifest=None, - add_conf_dirs=None): + add_conf_dirs=None, + cache_path_pattern=None): self.target_host = target_host self.hostdir = host_dir_name @@ -60,6 +63,7 @@ class Local(object): self.exec_path = exec_path self.custom_initial_manifest = initial_manifest self._add_conf_dirs = add_conf_dirs + self.cache_path_pattern = cache_path_pattern self._init_log() self._init_permissions() @@ -77,10 +81,7 @@ class Local(object): @property def home_dir(self): - if 'HOME' in os.environ: - return os.path.join(os.environ['HOME'], ".cdist") - else: - return None + return cdist.home_dir() def _init_log(self): self.log = logging.getLogger(self.target_host[0]) @@ -239,28 +240,70 @@ class Local(object): """ if os.access(script, os.X_OK): self.log.debug('%s is executable, running it', script) - command=[script] + command = [script] else: command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"] self.log.debug('%s is NOT executable, running it with %s', - script, " ".join(command)) + script, " ".join(command)) command.append(script) return self.run(command=command, env=env, return_output=return_output, message_prefix=message_prefix, save_output=save_output) - def save_cache(self): - destination = os.path.join(self.cache_path, self.hostdir) + def _cache_subpath_repl(self, matchobj): + if matchobj.group(2) == '%P': + repl = str(os.getpid()) + elif matchobj.group(2) == '%h': + repl = self.hostdir + elif matchobj.group(2) == '%N': + repl = self.target_host[0] + + return matchobj.group(1) + repl + + def _cache_subpath(self, start_time=time.time(), path_format=None): + if path_format: + repl_func = self._cache_subpath_repl + cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func, + path_format) + dt = datetime.datetime.fromtimestamp(start_time) + cache_subpath = dt.strftime(cache_subpath) + else: + cache_subpath = self.hostdir + + i = 0 + while i < len(cache_subpath) and cache_subpath[i] == os.sep: + i += 1 + cache_subpath = cache_subpath[i:] + if not cache_subpath: + cache_subpath = self.hostdir + return cache_subpath + + def save_cache(self, start_time=time.time()): + self.log.debug("cache subpath pattern: {}".format( + self.cache_path_pattern)) + cache_subpath = self._cache_subpath(start_time, + self.cache_path_pattern) + self.log.debug("cache subpath: {}".format(cache_subpath)) + destination = os.path.join(self.cache_path, cache_subpath) self.log.debug("Saving " + self.base_path + " to " + destination) - try: - if os.path.exists(destination): - shutil.rmtree(destination) - except PermissionError as e: - raise cdist.Error( - "Cannot delete old cache %s: %s" % (destination, e)) + if not os.path.exists(destination): + shutil.move(self.base_path, destination) + else: + for direntry in os.listdir(self.base_path): + srcentry = os.path.join(self.base_path, direntry) + destentry = os.path.join(destination, direntry) + try: + if os.path.isdir(destentry): + shutil.rmtree(destentry) + elif os.path.exists(destentry): + os.remove(destentry) + except (PermissionError, OSError) as e: + raise cdist.Error( + "Cannot delete old cache entry {}: {}".format( + destentry, e)) + shutil.move(srcentry, destentry) - shutil.move(self.base_path, destination) # add target_host since cache dir can be hash-ed target_host host_cache_path = os.path.join(destination, "target_host") with open(host_cache_path, 'w') as hostf: diff --git a/cdist/hostsource.py b/cdist/hostsource.py index 9c2c0616..a7b8f0b4 100644 --- a/cdist/hostsource.py +++ b/cdist/hostsource.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -22,6 +22,25 @@ import fileinput +def hostfile_process_line(line, strip_func=str.strip): + """Return entry from read line or None if no entry present.""" + if not line: + return None + # remove comment if present + comment_index = line.find('#') + if comment_index >= 0: + foo = line[:comment_index] + else: + foo = line + # remove leading and trailing whitespaces + foo = strip_func(foo) + # skip empty lines + if foo: + return foo + else: + return None + + class HostSource(object): """ Host source object. @@ -32,22 +51,7 @@ class HostSource(object): self.source = source def _process_file_line(self, line): - """Return host from read line or None if no host present.""" - if not line: - return None - # remove comment if present - comment_index = line.find('#') - if comment_index >= 0: - host = line[:comment_index] - else: - host = line - # remove leading and trailing whitespaces - host = host.strip() - # skip empty lines - if host: - return host - else: - return None + return hostfile_process_line(line) def _hosts_from_sequence(self): for host in self.source: diff --git a/cdist/test/exec/local.py b/cdist/test/exec/local.py index 0efdfa0a..6336947c 100644 --- a/cdist/test/exec/local.py +++ b/cdist/test/exec/local.py @@ -26,6 +26,8 @@ import getpass import shutil import string import random +import time +import datetime import cdist from cdist import test @@ -224,6 +226,41 @@ class LocalTestCase(test.CdistTestCase): self.assertTrue(os.path.isdir(self.local.bin_path)) self.assertTrue(os.path.isdir(self.local.conf_path)) + def test_cache_subpath(self): + start_time = time.time() + dt = datetime.datetime.fromtimestamp(start_time) + pid = str(os.getpid()) + cases = [ + ['', self.local.hostdir, ], + ['/', self.local.hostdir, ], + ['//', self.local.hostdir, ], + ['/%%h', '%h', ], + ['%%h', '%h', ], + ['%P', pid, ], + ['x%P', 'x' + pid, ], + ['%h', self.hostdir, ], + ['%h/%Y-%m-%d/%H%M%S%f%P', + dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ], + ['/%h/%Y-%m-%d/%H%M%S%f%P', + dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ], + ['%Y-%m-%d/%H%M%S%f%P/%h', + dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ], + ['///%Y-%m-%d/%H%M%S%f%P/%h', + dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ], + ['%h/%Y-%m-%d/%H%M%S-%P', + dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ], + ['%Y-%m-%d/%H%M%S-%P/%h', + dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ], + ['%N', self.local.target_host[0], ], + ] + for x in cases: + x.append(self.local._cache_subpath(start_time, x[0])) + # for fmt, expected, actual in cases: + # print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual)) + for fmt, expected, actual in cases: + self.assertEqual(expected, actual) + + if __name__ == "__main__": import unittest diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index 1c4226c2..c6066ffb 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -37,7 +37,7 @@ _cdist() ;; config|install) opts="-h --help -d --debug -v --verbose -b --beta \ - -c --conf-dir -f --file -i --initial-manifest -j --jobs \ + -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \ -n --dry-run -o --out-dir -p --parallel -s --sequential \ --remote-copy --remote-exec" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 001356d4..f56c4ac7 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -36,7 +36,7 @@ _cdist() esac ;; config|install) - opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) + opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) compadd "$@" -- $opts ;; *) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 4b94b858..5889ded9 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -276,4 +276,7 @@ CDIST_REMOTE_COPY CDIST_BETA Enable beta functionalities. + +CDIST_CACHE_PATH_PATTERN + Custom cache path pattern. eof diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 7112d14d..73cbbd12 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -15,13 +15,13 @@ SYNOPSIS cdist banner [-h] [-d] [-v] - cdist config [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] - [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] + cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + [-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [host [host ...]] - cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] - [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] + cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] + [-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [host [host ...]] @@ -76,6 +76,13 @@ Configure/install one or more hosts. Can also be enabled using CDIST_BETA env var. +.. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN + + Sepcify custom cache path pattern. It can also be set by + CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then + default hostdir is used. For more info on format see + :strong:`CACHE PATH PATTERN FORMAT` below. + .. option:: -c CONF_DIR, --conf-dir CONF_DIR Add a configuration directory. Can be specified multiple times. @@ -91,7 +98,8 @@ Configure/install one or more hosts. Read specified file for a list of additional hosts to operate on or if '-' is given, read stdin (one host per line). If no host or host file is specified then, by default, - read hosts from stdin. For the file format see below. + read hosts from stdin. For the file format see + :strong:`HOSTFILE FORMAT` below. .. option:: -i MANIFEST, --initial-manifest MANIFEST @@ -145,6 +153,24 @@ removed. Then all leading and trailing whitespace characters are stripped. If such a line results in empty line it is ignored/skipped. Otherwise, host string is used. +CACHE PATH PATTERN FORMAT +~~~~~~~~~~~~~~~~~~~~~~~~~ +Cache path pattern specifies path for a cache directory subdirectory. +In the path, '%N' will be substituted by the target host, '%h' will +be substituted by the calculated host directory, '%P' will be substituted +by the current process id. All format codes that +:strong:`python` :strong:`datetime.strftime()` function supports, except +'%h', are supported. These date/time directives format cdist config/install +start time. + +If empty pattern is specified then default calculated host directory +is used. + +Calculated host directory is a hash of a host cdist operates on. + +Resulting path is used to specify cache path subdirectory under which +current host cache data are saved. + SHELL ----- @@ -247,6 +273,9 @@ CDIST_REMOTE_COPY CDIST_BETA Enable beta functionality. +CDIST_CACHE_PATH_PATTERN + Custom cache path pattern. + EXIT STATUS ----------- The following exit values shall be returned: @@ -261,6 +290,7 @@ AUTHORS Originally written by Nico Schottelius and Steven Armstrong . + CAVEATS ------- When operating in parallel, either by operating in parallel for each host