Merge custom cache path pattern from beta branch.
This commit is contained in:
Darko Poljak
parent
b7873abf07
commit
2a9bd77550
10 changed files with 176 additions and 43 deletions
|
|
@ -114,3 +114,10 @@ def str_hash(s):
|
|||
return hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||
else:
|
||||
raise Error("Param should be string")
|
||||
|
||||
|
||||
def home_dir():
|
||||
if 'HOME' in os.environ:
|
||||
return os.path.join(os.environ['HOME'], ".cdist")
|
||||
else:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -116,6 +116,13 @@ def get_parsers():
|
|||
|
||||
# Config
|
||||
parser['config_main'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['config_main'].add_argument(
|
||||
'-C', '--cache-path-pattern',
|
||||
help=('Specify custom cache path pattern. It can also be set '
|
||||
'by CDIST_CACHE_PATH_PATTERN environment variable. If '
|
||||
'it is not set then default hostdir is used.'),
|
||||
dest='cache_path_pattern',
|
||||
default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
|
||||
parser['config_main'].add_argument(
|
||||
'-c', '--conf-dir',
|
||||
help=('Add configuration directory (can be repeated, '
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
|
||||
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
|
|
@ -223,7 +224,8 @@ class Config(object):
|
|||
base_root_path=host_base_path,
|
||||
host_dir_name=host_dir_name,
|
||||
initial_manifest=args.manifest,
|
||||
add_conf_dirs=args.conf_dir)
|
||||
add_conf_dirs=args.conf_dir,
|
||||
cache_path_pattern=args.cache_path_pattern)
|
||||
|
||||
remote = cdist.exec.remote.Remote(
|
||||
target_host=target_host,
|
||||
|
|
@ -260,7 +262,7 @@ class Config(object):
|
|||
self.manifest.run_initial_manifest(self.local.initial_manifest)
|
||||
self.iterate_until_finished()
|
||||
|
||||
self.local.save_cache()
|
||||
self.local.save_cache(start_time)
|
||||
self.log.info("Finished successful run in %s seconds",
|
||||
time.time() - start_time)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
#
|
||||
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
|
||||
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
|
|
@ -29,6 +29,8 @@ import subprocess
|
|||
import shutil
|
||||
import logging
|
||||
import tempfile
|
||||
import time
|
||||
import datetime
|
||||
|
||||
import cdist
|
||||
import cdist.message
|
||||
|
|
@ -51,7 +53,8 @@ class Local(object):
|
|||
host_dir_name,
|
||||
exec_path=sys.argv[0],
|
||||
initial_manifest=None,
|
||||
add_conf_dirs=None):
|
||||
add_conf_dirs=None,
|
||||
cache_path_pattern=None):
|
||||
|
||||
self.target_host = target_host
|
||||
self.hostdir = host_dir_name
|
||||
|
|
@ -60,6 +63,7 @@ class Local(object):
|
|||
self.exec_path = exec_path
|
||||
self.custom_initial_manifest = initial_manifest
|
||||
self._add_conf_dirs = add_conf_dirs
|
||||
self.cache_path_pattern = cache_path_pattern
|
||||
|
||||
self._init_log()
|
||||
self._init_permissions()
|
||||
|
|
@ -77,10 +81,7 @@ class Local(object):
|
|||
|
||||
@property
|
||||
def home_dir(self):
|
||||
if 'HOME' in os.environ:
|
||||
return os.path.join(os.environ['HOME'], ".cdist")
|
||||
else:
|
||||
return None
|
||||
return cdist.home_dir()
|
||||
|
||||
def _init_log(self):
|
||||
self.log = logging.getLogger(self.target_host[0])
|
||||
|
|
@ -239,28 +240,70 @@ class Local(object):
|
|||
"""
|
||||
if os.access(script, os.X_OK):
|
||||
self.log.debug('%s is executable, running it', script)
|
||||
command=[script]
|
||||
command = [script]
|
||||
else:
|
||||
command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
|
||||
self.log.debug('%s is NOT executable, running it with %s',
|
||||
script, " ".join(command))
|
||||
script, " ".join(command))
|
||||
command.append(script)
|
||||
|
||||
return self.run(command=command, env=env, return_output=return_output,
|
||||
message_prefix=message_prefix, save_output=save_output)
|
||||
|
||||
def save_cache(self):
|
||||
destination = os.path.join(self.cache_path, self.hostdir)
|
||||
def _cache_subpath_repl(self, matchobj):
|
||||
if matchobj.group(2) == '%P':
|
||||
repl = str(os.getpid())
|
||||
elif matchobj.group(2) == '%h':
|
||||
repl = self.hostdir
|
||||
elif matchobj.group(2) == '%N':
|
||||
repl = self.target_host[0]
|
||||
|
||||
return matchobj.group(1) + repl
|
||||
|
||||
def _cache_subpath(self, start_time=time.time(), path_format=None):
|
||||
if path_format:
|
||||
repl_func = self._cache_subpath_repl
|
||||
cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func,
|
||||
path_format)
|
||||
dt = datetime.datetime.fromtimestamp(start_time)
|
||||
cache_subpath = dt.strftime(cache_subpath)
|
||||
else:
|
||||
cache_subpath = self.hostdir
|
||||
|
||||
i = 0
|
||||
while i < len(cache_subpath) and cache_subpath[i] == os.sep:
|
||||
i += 1
|
||||
cache_subpath = cache_subpath[i:]
|
||||
if not cache_subpath:
|
||||
cache_subpath = self.hostdir
|
||||
return cache_subpath
|
||||
|
||||
def save_cache(self, start_time=time.time()):
|
||||
self.log.debug("cache subpath pattern: {}".format(
|
||||
self.cache_path_pattern))
|
||||
cache_subpath = self._cache_subpath(start_time,
|
||||
self.cache_path_pattern)
|
||||
self.log.debug("cache subpath: {}".format(cache_subpath))
|
||||
destination = os.path.join(self.cache_path, cache_subpath)
|
||||
self.log.debug("Saving " + self.base_path + " to " + destination)
|
||||
|
||||
try:
|
||||
if os.path.exists(destination):
|
||||
shutil.rmtree(destination)
|
||||
except PermissionError as e:
|
||||
raise cdist.Error(
|
||||
"Cannot delete old cache %s: %s" % (destination, e))
|
||||
if not os.path.exists(destination):
|
||||
shutil.move(self.base_path, destination)
|
||||
else:
|
||||
for direntry in os.listdir(self.base_path):
|
||||
srcentry = os.path.join(self.base_path, direntry)
|
||||
destentry = os.path.join(destination, direntry)
|
||||
try:
|
||||
if os.path.isdir(destentry):
|
||||
shutil.rmtree(destentry)
|
||||
elif os.path.exists(destentry):
|
||||
os.remove(destentry)
|
||||
except (PermissionError, OSError) as e:
|
||||
raise cdist.Error(
|
||||
"Cannot delete old cache entry {}: {}".format(
|
||||
destentry, e))
|
||||
shutil.move(srcentry, destentry)
|
||||
|
||||
shutil.move(self.base_path, destination)
|
||||
# add target_host since cache dir can be hash-ed target_host
|
||||
host_cache_path = os.path.join(destination, "target_host")
|
||||
with open(host_cache_path, 'w') as hostf:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
|
|
@ -22,6 +22,25 @@
|
|||
import fileinput
|
||||
|
||||
|
||||
def hostfile_process_line(line, strip_func=str.strip):
|
||||
"""Return entry from read line or None if no entry present."""
|
||||
if not line:
|
||||
return None
|
||||
# remove comment if present
|
||||
comment_index = line.find('#')
|
||||
if comment_index >= 0:
|
||||
foo = line[:comment_index]
|
||||
else:
|
||||
foo = line
|
||||
# remove leading and trailing whitespaces
|
||||
foo = strip_func(foo)
|
||||
# skip empty lines
|
||||
if foo:
|
||||
return foo
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class HostSource(object):
|
||||
"""
|
||||
Host source object.
|
||||
|
|
@ -32,22 +51,7 @@ class HostSource(object):
|
|||
self.source = source
|
||||
|
||||
def _process_file_line(self, line):
|
||||
"""Return host from read line or None if no host present."""
|
||||
if not line:
|
||||
return None
|
||||
# remove comment if present
|
||||
comment_index = line.find('#')
|
||||
if comment_index >= 0:
|
||||
host = line[:comment_index]
|
||||
else:
|
||||
host = line
|
||||
# remove leading and trailing whitespaces
|
||||
host = host.strip()
|
||||
# skip empty lines
|
||||
if host:
|
||||
return host
|
||||
else:
|
||||
return None
|
||||
return hostfile_process_line(line)
|
||||
|
||||
def _hosts_from_sequence(self):
|
||||
for host in self.source:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ import getpass
|
|||
import shutil
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import datetime
|
||||
|
||||
import cdist
|
||||
from cdist import test
|
||||
|
|
@ -224,6 +226,41 @@ class LocalTestCase(test.CdistTestCase):
|
|||
self.assertTrue(os.path.isdir(self.local.bin_path))
|
||||
self.assertTrue(os.path.isdir(self.local.conf_path))
|
||||
|
||||
def test_cache_subpath(self):
|
||||
start_time = time.time()
|
||||
dt = datetime.datetime.fromtimestamp(start_time)
|
||||
pid = str(os.getpid())
|
||||
cases = [
|
||||
['', self.local.hostdir, ],
|
||||
['/', self.local.hostdir, ],
|
||||
['//', self.local.hostdir, ],
|
||||
['/%%h', '%h', ],
|
||||
['%%h', '%h', ],
|
||||
['%P', pid, ],
|
||||
['x%P', 'x' + pid, ],
|
||||
['%h', self.hostdir, ],
|
||||
['%h/%Y-%m-%d/%H%M%S%f%P',
|
||||
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
|
||||
['/%h/%Y-%m-%d/%H%M%S%f%P',
|
||||
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
|
||||
['%Y-%m-%d/%H%M%S%f%P/%h',
|
||||
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
|
||||
['///%Y-%m-%d/%H%M%S%f%P/%h',
|
||||
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
|
||||
['%h/%Y-%m-%d/%H%M%S-%P',
|
||||
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ],
|
||||
['%Y-%m-%d/%H%M%S-%P/%h',
|
||||
dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ],
|
||||
['%N', self.local.target_host[0], ],
|
||||
]
|
||||
for x in cases:
|
||||
x.append(self.local._cache_subpath(start_time, x[0]))
|
||||
# for fmt, expected, actual in cases:
|
||||
# print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual))
|
||||
for fmt, expected, actual in cases:
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import unittest
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ _cdist()
|
|||
;;
|
||||
config|install)
|
||||
opts="-h --help -d --debug -v --verbose -b --beta \
|
||||
-c --conf-dir -f --file -i --initial-manifest -j --jobs \
|
||||
-C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \
|
||||
-n --dry-run -o --out-dir -p --parallel -s --sequential \
|
||||
--remote-copy --remote-exec"
|
||||
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )
|
||||
|
|
|
|||
|
|
@ -36,7 +36,7 @@ _cdist()
|
|||
esac
|
||||
;;
|
||||
config|install)
|
||||
opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
|
||||
opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
|
||||
compadd "$@" -- $opts
|
||||
;;
|
||||
*)
|
||||
|
|
|
|||
|
|
@ -276,4 +276,7 @@ CDIST_REMOTE_COPY
|
|||
|
||||
CDIST_BETA
|
||||
Enable beta functionalities.
|
||||
|
||||
CDIST_CACHE_PATH_PATTERN
|
||||
Custom cache path pattern.
|
||||
eof
|
||||
|
|
|
|||
|
|
@ -15,13 +15,13 @@ SYNOPSIS
|
|||
|
||||
cdist banner [-h] [-d] [-v]
|
||||
|
||||
cdist config [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE]
|
||||
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s]
|
||||
cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
|
||||
[-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
|
||||
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
|
||||
[host [host ...]]
|
||||
|
||||
cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE]
|
||||
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s]
|
||||
cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
|
||||
[-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
|
||||
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
|
||||
[host [host ...]]
|
||||
|
||||
|
|
@ -76,6 +76,13 @@ Configure/install one or more hosts.
|
|||
|
||||
Can also be enabled using CDIST_BETA env var.
|
||||
|
||||
.. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN
|
||||
|
||||
Sepcify custom cache path pattern. It can also be set by
|
||||
CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
|
||||
default hostdir is used. For more info on format see
|
||||
:strong:`CACHE PATH PATTERN FORMAT` below.
|
||||
|
||||
.. option:: -c CONF_DIR, --conf-dir CONF_DIR
|
||||
|
||||
Add a configuration directory. Can be specified multiple times.
|
||||
|
|
@ -91,7 +98,8 @@ Configure/install one or more hosts.
|
|||
Read specified file for a list of additional hosts to operate on
|
||||
or if '-' is given, read stdin (one host per line).
|
||||
If no host or host file is specified then, by default,
|
||||
read hosts from stdin. For the file format see below.
|
||||
read hosts from stdin. For the file format see
|
||||
:strong:`HOSTFILE FORMAT` below.
|
||||
|
||||
.. option:: -i MANIFEST, --initial-manifest MANIFEST
|
||||
|
||||
|
|
@ -145,6 +153,24 @@ removed. Then all leading and trailing whitespace characters are stripped.
|
|||
If such a line results in empty line it is ignored/skipped. Otherwise,
|
||||
host string is used.
|
||||
|
||||
CACHE PATH PATTERN FORMAT
|
||||
~~~~~~~~~~~~~~~~~~~~~~~~~
|
||||
Cache path pattern specifies path for a cache directory subdirectory.
|
||||
In the path, '%N' will be substituted by the target host, '%h' will
|
||||
be substituted by the calculated host directory, '%P' will be substituted
|
||||
by the current process id. All format codes that
|
||||
:strong:`python` :strong:`datetime.strftime()` function supports, except
|
||||
'%h', are supported. These date/time directives format cdist config/install
|
||||
start time.
|
||||
|
||||
If empty pattern is specified then default calculated host directory
|
||||
is used.
|
||||
|
||||
Calculated host directory is a hash of a host cdist operates on.
|
||||
|
||||
Resulting path is used to specify cache path subdirectory under which
|
||||
current host cache data are saved.
|
||||
|
||||
|
||||
SHELL
|
||||
-----
|
||||
|
|
@ -247,6 +273,9 @@ CDIST_REMOTE_COPY
|
|||
CDIST_BETA
|
||||
Enable beta functionality.
|
||||
|
||||
CDIST_CACHE_PATH_PATTERN
|
||||
Custom cache path pattern.
|
||||
|
||||
EXIT STATUS
|
||||
-----------
|
||||
The following exit values shall be returned:
|
||||
|
|
@ -261,6 +290,7 @@ AUTHORS
|
|||
Originally written by Nico Schottelius <nico-cdist--@--schottelius.org>
|
||||
and Steven Armstrong <steven-cdist--@--armstrong.cc>.
|
||||
|
||||
|
||||
CAVEATS
|
||||
-------
|
||||
When operating in parallel, either by operating in parallel for each host
|
||||
|
|
|
|||
Loading…
Reference in a new issue