Merge custom cache path pattern from beta branch.

cdist/__init__.py

@@ -114,3 +114,10 @@ def str_hash(s):
         return hashlib.md5(s.encode('utf-8')).hexdigest()
     else:
         raise Error("Param should be string")
+
+
+def home_dir():
+    if 'HOME' in os.environ:
+        return os.path.join(os.environ['HOME'], ".cdist")
+    else:
+        return None

cdist/argparse.py

@@ -116,6 +116,13 @@ def get_parsers():
 
     # Config
     parser['config_main'] = argparse.ArgumentParser(add_help=False)
+    parser['config_main'].add_argument(
+            '-C', '--cache-path-pattern',
+            help=('Specify custom cache path pattern. It can also be set '
+                  'by CDIST_CACHE_PATH_PATTERN environment variable. If '
+                  'it is not set then default hostdir is used.'),
+            dest='cache_path_pattern',
+            default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
     parser['config_main'].add_argument(
             '-c', '--conf-dir',
             help=('Add configuration directory (can be repeated, '

cdist/config.py

@@ -2,6 +2,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
+# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -223,7 +224,8 @@ class Config(object):
                 base_root_path=host_base_path,
                 host_dir_name=host_dir_name,
                 initial_manifest=args.manifest,
-                add_conf_dirs=args.conf_dir)
+                add_conf_dirs=args.conf_dir,
+                cache_path_pattern=args.cache_path_pattern)
 
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
@@ -260,7 +262,7 @@ class Config(object):
         self.manifest.run_initial_manifest(self.local.initial_manifest)
         self.iterate_until_finished()
 
-        self.local.save_cache()
+        self.local.save_cache(start_time)
         self.log.info("Finished successful run in %s seconds",
                       time.time() - start_time)
 

cdist/exec/local.py

@@ -2,7 +2,7 @@
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
-# 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -29,6 +29,8 @@ import subprocess
 import shutil
 import logging
 import tempfile
+import time
+import datetime
 
 import cdist
 import cdist.message
@@ -51,7 +53,8 @@ class Local(object):
                  host_dir_name,
                  exec_path=sys.argv[0],
                  initial_manifest=None,
-                 add_conf_dirs=None):
+                 add_conf_dirs=None,
+                 cache_path_pattern=None):
 
         self.target_host = target_host
         self.hostdir = host_dir_name
@@ -60,6 +63,7 @@ class Local(object):
         self.exec_path = exec_path
         self.custom_initial_manifest = initial_manifest
         self._add_conf_dirs = add_conf_dirs
+        self.cache_path_pattern = cache_path_pattern
 
         self._init_log()
         self._init_permissions()
@@ -77,10 +81,7 @@ class Local(object):
 
     @property
     def home_dir(self):
-        if 'HOME' in os.environ:
+        return cdist.home_dir()
-            return os.path.join(os.environ['HOME'], ".cdist")
-        else:
-            return None
 
     def _init_log(self):
         self.log = logging.getLogger(self.target_host[0])
@@ -239,7 +240,7 @@ class Local(object):
         """
         if os.access(script, os.X_OK):
             self.log.debug('%s is executable, running it', script)
-            command=[script]
+            command = [script]
         else:
             command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
             self.log.debug('%s is NOT executable, running it with %s',
@@ -249,18 +250,60 @@ class Local(object):
         return self.run(command=command, env=env, return_output=return_output,
                         message_prefix=message_prefix, save_output=save_output)
 
-    def save_cache(self):
+    def _cache_subpath_repl(self, matchobj):
-        destination = os.path.join(self.cache_path, self.hostdir)
+        if matchobj.group(2) == '%P':
+            repl = str(os.getpid())
+        elif matchobj.group(2) == '%h':
+            repl = self.hostdir
+        elif matchobj.group(2) == '%N':
+            repl = self.target_host[0]
+
+        return matchobj.group(1) + repl
+
+    def _cache_subpath(self, start_time=time.time(), path_format=None):
+        if path_format:
+            repl_func = self._cache_subpath_repl
+            cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func,
+                                   path_format)
+            dt = datetime.datetime.fromtimestamp(start_time)
+            cache_subpath = dt.strftime(cache_subpath)
+        else:
+            cache_subpath = self.hostdir
+
+        i = 0
+        while i < len(cache_subpath) and cache_subpath[i] == os.sep:
+            i += 1
+        cache_subpath = cache_subpath[i:]
+        if not cache_subpath:
+            cache_subpath = self.hostdir
+        return cache_subpath
+
+    def save_cache(self, start_time=time.time()):
+        self.log.debug("cache subpath pattern: {}".format(
+            self.cache_path_pattern))
+        cache_subpath = self._cache_subpath(start_time,
+                                            self.cache_path_pattern)
+        self.log.debug("cache subpath: {}".format(cache_subpath))
+        destination = os.path.join(self.cache_path, cache_subpath)
         self.log.debug("Saving " + self.base_path + " to " + destination)
 
-        try:
+        if not os.path.exists(destination):
-            if os.path.exists(destination):
-                shutil.rmtree(destination)
-        except PermissionError as e:
-            raise cdist.Error(
-                    "Cannot delete old cache %s: %s" % (destination, e))
-
             shutil.move(self.base_path, destination)
+        else:
+            for direntry in os.listdir(self.base_path):
+                srcentry = os.path.join(self.base_path, direntry)
+                destentry = os.path.join(destination, direntry)
+                try:
+                    if os.path.isdir(destentry):
+                        shutil.rmtree(destentry)
+                    elif os.path.exists(destentry):
+                        os.remove(destentry)
+                except (PermissionError, OSError) as e:
+                    raise cdist.Error(
+                            "Cannot delete old cache entry {}: {}".format(
+                                destentry, e))
+                shutil.move(srcentry, destentry)
+
         # add target_host since cache dir can be hash-ed target_host
         host_cache_path = os.path.join(destination, "target_host")
         with open(host_cache_path, 'w') as hostf:

cdist/hostsource.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 #
-# 2016 Darko Poljak (darko.poljak at gmail.com)
+# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
 #
 # This file is part of cdist.
 #
@@ -22,6 +22,25 @@
 import fileinput
 
 
+def hostfile_process_line(line, strip_func=str.strip):
+    """Return entry from read line or None if no entry present."""
+    if not line:
+        return None
+    # remove comment if present
+    comment_index = line.find('#')
+    if comment_index >= 0:
+        foo = line[:comment_index]
+    else:
+        foo = line
+    # remove leading and trailing whitespaces
+    foo = strip_func(foo)
+    # skip empty lines
+    if foo:
+        return foo
+    else:
+        return None
+
+
 class HostSource(object):
     """
     Host source object.
@@ -32,22 +51,7 @@ class HostSource(object):
         self.source = source
 
     def _process_file_line(self, line):
-        """Return host from read line or None if no host present."""
+        return hostfile_process_line(line)
-        if not line:
-            return None
-        # remove comment if present
-        comment_index = line.find('#')
-        if comment_index >= 0:
-            host = line[:comment_index]
-        else:
-            host = line
-        # remove leading and trailing whitespaces
-        host = host.strip()
-        # skip empty lines
-        if host:
-            return host
-        else:
-            return None
 
     def _hosts_from_sequence(self):
         for host in self.source:

cdist/test/exec/local.py

@@ -26,6 +26,8 @@ import getpass
 import shutil
 import string
 import random
+import time
+import datetime
 
 import cdist
 from cdist import test
@@ -224,6 +226,41 @@ class LocalTestCase(test.CdistTestCase):
         self.assertTrue(os.path.isdir(self.local.bin_path))
         self.assertTrue(os.path.isdir(self.local.conf_path))
 
+    def test_cache_subpath(self):
+        start_time = time.time()
+        dt = datetime.datetime.fromtimestamp(start_time)
+        pid = str(os.getpid())
+        cases = [
+            ['', self.local.hostdir, ],
+            ['/', self.local.hostdir, ],
+            ['//', self.local.hostdir, ],
+            ['/%%h', '%h', ],
+            ['%%h', '%h', ],
+            ['%P', pid, ],
+            ['x%P', 'x' + pid, ],
+            ['%h', self.hostdir, ],
+            ['%h/%Y-%m-%d/%H%M%S%f%P',
+             dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
+            ['/%h/%Y-%m-%d/%H%M%S%f%P',
+             dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
+            ['%Y-%m-%d/%H%M%S%f%P/%h',
+             dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
+            ['///%Y-%m-%d/%H%M%S%f%P/%h',
+             dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
+            ['%h/%Y-%m-%d/%H%M%S-%P',
+             dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ],
+            ['%Y-%m-%d/%H%M%S-%P/%h',
+             dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ],
+            ['%N', self.local.target_host[0], ],
+        ]
+        for x in cases:
+            x.append(self.local._cache_subpath(start_time, x[0]))
+        # for fmt, expected, actual in cases:
+        #     print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual))
+        for fmt, expected, actual in cases:
+            self.assertEqual(expected, actual)
+
+
 if __name__ == "__main__":
     import unittest
 

completions/bash/cdist-completion.bash

@@ -37,7 +37,7 @@ _cdist()
             ;;
         config|install)
             opts="-h --help -d --debug -v --verbose -b --beta \
-                -c --conf-dir -f --file -i --initial-manifest -j --jobs \
+                -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \
                 -n --dry-run -o --out-dir -p --parallel -s --sequential \
                 --remote-copy --remote-exec"
             COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )

completions/zsh/_cdist

@@ -36,7 +36,7 @@ _cdist()
                     esac
                     ;;
                 config|install)
-                    opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
+                    opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
                     compadd "$@" -- $opts
                     ;;
                *)

docs/src/cdist-reference.rst.sh

@@ -276,4 +276,7 @@ CDIST_REMOTE_COPY
 
 CDIST_BETA
     Enable beta functionalities.
+
+CDIST_CACHE_PATH_PATTERN
+    Custom cache path pattern.
 eof

docs/src/man1/cdist.rst

@@ -15,13 +15,13 @@ SYNOPSIS
 
     cdist banner [-h] [-d] [-v]
 
-    cdist config [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE]
+    cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
-                 [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s]
+                 [-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
                  [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
                  [host [host ...]]
 
-    cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE]
+    cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
-                  [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s]
+                  [-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
                   [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
                   [host [host ...]]
 
@@ -76,6 +76,13 @@ Configure/install one or more hosts.
     
     Can also be enabled using CDIST_BETA env var.
 
+.. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN
+
+    Sepcify custom cache path pattern. It can also be set by
+    CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
+    default hostdir is used. For more info on format see
+    :strong:`CACHE PATH PATTERN FORMAT` below.
+
 .. option:: -c CONF_DIR, --conf-dir CONF_DIR
 
     Add a configuration directory. Can be specified multiple times.
@@ -91,7 +98,8 @@ Configure/install one or more hosts.
     Read specified file for a list of additional hosts to operate on
     or if '-' is given, read stdin (one host per line).
     If no host or host file is specified then, by default,
-    read hosts from stdin. For the file format see below.
+    read hosts from stdin. For the file format see
+    :strong:`HOSTFILE FORMAT` below.
 
 .. option:: -i MANIFEST, --initial-manifest MANIFEST
 
@@ -145,6 +153,24 @@ removed. Then all leading and trailing whitespace characters are stripped.
 If such a line results in empty line it is ignored/skipped. Otherwise,
 host string is used.
 
+CACHE PATH PATTERN FORMAT
+~~~~~~~~~~~~~~~~~~~~~~~~~
+Cache path pattern specifies path for a cache directory subdirectory.
+In the path, '%N' will be substituted by the target host, '%h' will
+be substituted by the calculated host directory, '%P' will be substituted
+by the current process id. All format codes that
+:strong:`python` :strong:`datetime.strftime()` function supports, except
+'%h', are supported. These date/time directives format cdist config/install
+start time.
+
+If empty pattern is specified then default calculated host directory
+is used.
+
+Calculated host directory is a hash of a host cdist operates on.
+
+Resulting path is used to specify cache path subdirectory under which
+current host cache data are saved.
+
 
 SHELL
 -----
@@ -247,6 +273,9 @@ CDIST_REMOTE_COPY
 CDIST_BETA
     Enable beta functionality.
 
+CDIST_CACHE_PATH_PATTERN
+    Custom cache path pattern.
+
 EXIT STATUS
 -----------
 The following exit values shall be returned:
@@ -261,6 +290,7 @@ AUTHORS
 Originally written by Nico Schottelius <nico-cdist--@--schottelius.org>
 and Steven Armstrong <steven-cdist--@--armstrong.cc>.
 
+
 CAVEATS
 -------
 When operating in parallel, either by operating in parallel for each host