Merge custom cache path pattern from beta branch.

This commit is contained in:
Darko Poljak 2017-07-01 23:59:51 +02:00
parent b7873abf07
commit 2a9bd77550
10 changed files with 176 additions and 43 deletions

View File

@ -114,3 +114,10 @@ def str_hash(s):
return hashlib.md5(s.encode('utf-8')).hexdigest() return hashlib.md5(s.encode('utf-8')).hexdigest()
else: else:
raise Error("Param should be string") raise Error("Param should be string")
def home_dir():
if 'HOME' in os.environ:
return os.path.join(os.environ['HOME'], ".cdist")
else:
return None

View File

@ -116,6 +116,13 @@ def get_parsers():
# Config # Config
parser['config_main'] = argparse.ArgumentParser(add_help=False) parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
'-C', '--cache-path-pattern',
help=('Specify custom cache path pattern. It can also be set '
'by CDIST_CACHE_PATH_PATTERN environment variable. If '
'it is not set then default hostdir is used.'),
dest='cache_path_pattern',
default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
parser['config_main'].add_argument( parser['config_main'].add_argument(
'-c', '--conf-dir', '-c', '--conf-dir',
help=('Add configuration directory (can be repeated, ' help=('Add configuration directory (can be repeated, '

View File

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -223,7 +224,8 @@ class Config(object):
base_root_path=host_base_path, base_root_path=host_base_path,
host_dir_name=host_dir_name, host_dir_name=host_dir_name,
initial_manifest=args.manifest, initial_manifest=args.manifest,
add_conf_dirs=args.conf_dir) add_conf_dirs=args.conf_dir,
cache_path_pattern=args.cache_path_pattern)
remote = cdist.exec.remote.Remote( remote = cdist.exec.remote.Remote(
target_host=target_host, target_host=target_host,
@ -260,7 +262,7 @@ class Config(object):
self.manifest.run_initial_manifest(self.local.initial_manifest) self.manifest.run_initial_manifest(self.local.initial_manifest)
self.iterate_until_finished() self.iterate_until_finished()
self.local.save_cache() self.local.save_cache(start_time)
self.log.info("Finished successful run in %s seconds", self.log.info("Finished successful run in %s seconds",
time.time() - start_time) time.time() - start_time)

View File

@ -2,7 +2,7 @@
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -29,6 +29,8 @@ import subprocess
import shutil import shutil
import logging import logging
import tempfile import tempfile
import time
import datetime
import cdist import cdist
import cdist.message import cdist.message
@ -51,7 +53,8 @@ class Local(object):
host_dir_name, host_dir_name,
exec_path=sys.argv[0], exec_path=sys.argv[0],
initial_manifest=None, initial_manifest=None,
add_conf_dirs=None): add_conf_dirs=None,
cache_path_pattern=None):
self.target_host = target_host self.target_host = target_host
self.hostdir = host_dir_name self.hostdir = host_dir_name
@ -60,6 +63,7 @@ class Local(object):
self.exec_path = exec_path self.exec_path = exec_path
self.custom_initial_manifest = initial_manifest self.custom_initial_manifest = initial_manifest
self._add_conf_dirs = add_conf_dirs self._add_conf_dirs = add_conf_dirs
self.cache_path_pattern = cache_path_pattern
self._init_log() self._init_log()
self._init_permissions() self._init_permissions()
@ -77,10 +81,7 @@ class Local(object):
@property @property
def home_dir(self): def home_dir(self):
if 'HOME' in os.environ: return cdist.home_dir()
return os.path.join(os.environ['HOME'], ".cdist")
else:
return None
def _init_log(self): def _init_log(self):
self.log = logging.getLogger(self.target_host[0]) self.log = logging.getLogger(self.target_host[0])
@ -239,28 +240,70 @@ class Local(object):
""" """
if os.access(script, os.X_OK): if os.access(script, os.X_OK):
self.log.debug('%s is executable, running it', script) self.log.debug('%s is executable, running it', script)
command=[script] command = [script]
else: else:
command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"] command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
self.log.debug('%s is NOT executable, running it with %s', self.log.debug('%s is NOT executable, running it with %s',
script, " ".join(command)) script, " ".join(command))
command.append(script) command.append(script)
return self.run(command=command, env=env, return_output=return_output, return self.run(command=command, env=env, return_output=return_output,
message_prefix=message_prefix, save_output=save_output) message_prefix=message_prefix, save_output=save_output)
def save_cache(self): def _cache_subpath_repl(self, matchobj):
destination = os.path.join(self.cache_path, self.hostdir) if matchobj.group(2) == '%P':
repl = str(os.getpid())
elif matchobj.group(2) == '%h':
repl = self.hostdir
elif matchobj.group(2) == '%N':
repl = self.target_host[0]
return matchobj.group(1) + repl
def _cache_subpath(self, start_time=time.time(), path_format=None):
if path_format:
repl_func = self._cache_subpath_repl
cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func,
path_format)
dt = datetime.datetime.fromtimestamp(start_time)
cache_subpath = dt.strftime(cache_subpath)
else:
cache_subpath = self.hostdir
i = 0
while i < len(cache_subpath) and cache_subpath[i] == os.sep:
i += 1
cache_subpath = cache_subpath[i:]
if not cache_subpath:
cache_subpath = self.hostdir
return cache_subpath
def save_cache(self, start_time=time.time()):
self.log.debug("cache subpath pattern: {}".format(
self.cache_path_pattern))
cache_subpath = self._cache_subpath(start_time,
self.cache_path_pattern)
self.log.debug("cache subpath: {}".format(cache_subpath))
destination = os.path.join(self.cache_path, cache_subpath)
self.log.debug("Saving " + self.base_path + " to " + destination) self.log.debug("Saving " + self.base_path + " to " + destination)
try: if not os.path.exists(destination):
if os.path.exists(destination): shutil.move(self.base_path, destination)
shutil.rmtree(destination) else:
except PermissionError as e: for direntry in os.listdir(self.base_path):
raise cdist.Error( srcentry = os.path.join(self.base_path, direntry)
"Cannot delete old cache %s: %s" % (destination, e)) destentry = os.path.join(destination, direntry)
try:
if os.path.isdir(destentry):
shutil.rmtree(destentry)
elif os.path.exists(destentry):
os.remove(destentry)
except (PermissionError, OSError) as e:
raise cdist.Error(
"Cannot delete old cache entry {}: {}".format(
destentry, e))
shutil.move(srcentry, destentry)
shutil.move(self.base_path, destination)
# add target_host since cache dir can be hash-ed target_host # add target_host since cache dir can be hash-ed target_host
host_cache_path = os.path.join(destination, "target_host") host_cache_path = os.path.join(destination, "target_host")
with open(host_cache_path, 'w') as hostf: with open(host_cache_path, 'w') as hostf:

View File

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2016 Darko Poljak (darko.poljak at gmail.com) # 2016-2017 Darko Poljak (darko.poljak at gmail.com)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -22,6 +22,25 @@
import fileinput import fileinput
def hostfile_process_line(line, strip_func=str.strip):
"""Return entry from read line or None if no entry present."""
if not line:
return None
# remove comment if present
comment_index = line.find('#')
if comment_index >= 0:
foo = line[:comment_index]
else:
foo = line
# remove leading and trailing whitespaces
foo = strip_func(foo)
# skip empty lines
if foo:
return foo
else:
return None
class HostSource(object): class HostSource(object):
""" """
Host source object. Host source object.
@ -32,22 +51,7 @@ class HostSource(object):
self.source = source self.source = source
def _process_file_line(self, line): def _process_file_line(self, line):
"""Return host from read line or None if no host present.""" return hostfile_process_line(line)
if not line:
return None
# remove comment if present
comment_index = line.find('#')
if comment_index >= 0:
host = line[:comment_index]
else:
host = line
# remove leading and trailing whitespaces
host = host.strip()
# skip empty lines
if host:
return host
else:
return None
def _hosts_from_sequence(self): def _hosts_from_sequence(self):
for host in self.source: for host in self.source:

View File

@ -26,6 +26,8 @@ import getpass
import shutil import shutil
import string import string
import random import random
import time
import datetime
import cdist import cdist
from cdist import test from cdist import test
@ -224,6 +226,41 @@ class LocalTestCase(test.CdistTestCase):
self.assertTrue(os.path.isdir(self.local.bin_path)) self.assertTrue(os.path.isdir(self.local.bin_path))
self.assertTrue(os.path.isdir(self.local.conf_path)) self.assertTrue(os.path.isdir(self.local.conf_path))
def test_cache_subpath(self):
start_time = time.time()
dt = datetime.datetime.fromtimestamp(start_time)
pid = str(os.getpid())
cases = [
['', self.local.hostdir, ],
['/', self.local.hostdir, ],
['//', self.local.hostdir, ],
['/%%h', '%h', ],
['%%h', '%h', ],
['%P', pid, ],
['x%P', 'x' + pid, ],
['%h', self.hostdir, ],
['%h/%Y-%m-%d/%H%M%S%f%P',
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
['/%h/%Y-%m-%d/%H%M%S%f%P',
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
['%Y-%m-%d/%H%M%S%f%P/%h',
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
['///%Y-%m-%d/%H%M%S%f%P/%h',
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
['%h/%Y-%m-%d/%H%M%S-%P',
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ],
['%Y-%m-%d/%H%M%S-%P/%h',
dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ],
['%N', self.local.target_host[0], ],
]
for x in cases:
x.append(self.local._cache_subpath(start_time, x[0]))
# for fmt, expected, actual in cases:
# print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual))
for fmt, expected, actual in cases:
self.assertEqual(expected, actual)
if __name__ == "__main__": if __name__ == "__main__":
import unittest import unittest

View File

@ -37,7 +37,7 @@ _cdist()
;; ;;
config|install) config|install)
opts="-h --help -d --debug -v --verbose -b --beta \ opts="-h --help -d --debug -v --verbose -b --beta \
-c --conf-dir -f --file -i --initial-manifest -j --jobs \ -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \
-n --dry-run -o --out-dir -p --parallel -s --sequential \ -n --dry-run -o --out-dir -p --parallel -s --sequential \
--remote-copy --remote-exec" --remote-copy --remote-exec"
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )

View File

@ -36,7 +36,7 @@ _cdist()
esac esac
;; ;;
config|install) config|install)
opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
compadd "$@" -- $opts compadd "$@" -- $opts
;; ;;
*) *)

View File

@ -276,4 +276,7 @@ CDIST_REMOTE_COPY
CDIST_BETA CDIST_BETA
Enable beta functionalities. Enable beta functionalities.
CDIST_CACHE_PATH_PATTERN
Custom cache path pattern.
eof eof

View File

@ -15,13 +15,13 @@ SYNOPSIS
cdist banner [-h] [-d] [-v] cdist banner [-h] [-d] [-v]
cdist config [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] [-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[host [host ...]] [host [host ...]]
cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] [-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[host [host ...]] [host [host ...]]
@ -76,6 +76,13 @@ Configure/install one or more hosts.
Can also be enabled using CDIST_BETA env var. Can also be enabled using CDIST_BETA env var.
.. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN
Sepcify custom cache path pattern. It can also be set by
CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
default hostdir is used. For more info on format see
:strong:`CACHE PATH PATTERN FORMAT` below.
.. option:: -c CONF_DIR, --conf-dir CONF_DIR .. option:: -c CONF_DIR, --conf-dir CONF_DIR
Add a configuration directory. Can be specified multiple times. Add a configuration directory. Can be specified multiple times.
@ -91,7 +98,8 @@ Configure/install one or more hosts.
Read specified file for a list of additional hosts to operate on Read specified file for a list of additional hosts to operate on
or if '-' is given, read stdin (one host per line). or if '-' is given, read stdin (one host per line).
If no host or host file is specified then, by default, If no host or host file is specified then, by default,
read hosts from stdin. For the file format see below. read hosts from stdin. For the file format see
:strong:`HOSTFILE FORMAT` below.
.. option:: -i MANIFEST, --initial-manifest MANIFEST .. option:: -i MANIFEST, --initial-manifest MANIFEST
@ -145,6 +153,24 @@ removed. Then all leading and trailing whitespace characters are stripped.
If such a line results in empty line it is ignored/skipped. Otherwise, If such a line results in empty line it is ignored/skipped. Otherwise,
host string is used. host string is used.
CACHE PATH PATTERN FORMAT
~~~~~~~~~~~~~~~~~~~~~~~~~
Cache path pattern specifies path for a cache directory subdirectory.
In the path, '%N' will be substituted by the target host, '%h' will
be substituted by the calculated host directory, '%P' will be substituted
by the current process id. All format codes that
:strong:`python` :strong:`datetime.strftime()` function supports, except
'%h', are supported. These date/time directives format cdist config/install
start time.
If empty pattern is specified then default calculated host directory
is used.
Calculated host directory is a hash of a host cdist operates on.
Resulting path is used to specify cache path subdirectory under which
current host cache data are saved.
SHELL SHELL
----- -----
@ -247,6 +273,9 @@ CDIST_REMOTE_COPY
CDIST_BETA CDIST_BETA
Enable beta functionality. Enable beta functionality.
CDIST_CACHE_PATH_PATTERN
Custom cache path pattern.
EXIT STATUS EXIT STATUS
----------- -----------
The following exit values shall be returned: The following exit values shall be returned:
@ -261,6 +290,7 @@ AUTHORS
Originally written by Nico Schottelius <nico-cdist--@--schottelius.org> Originally written by Nico Schottelius <nico-cdist--@--schottelius.org>
and Steven Armstrong <steven-cdist--@--armstrong.cc>. and Steven Armstrong <steven-cdist--@--armstrong.cc>.
CAVEATS CAVEATS
------- -------
When operating in parallel, either by operating in parallel for each host When operating in parallel, either by operating in parallel for each host