matthijs/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests Projects Releases Wiki Activity

Merge custom cache path pattern from beta branch.

Browse source
This commit is contained in:
Darko Poljak 2017-07-01 23:59:51 +02:00
commit 2a9bd77550
10 changed files with 176 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

7
cdist/__init__.py
View file

@ -114,3 +114,10 @@ def str_hash(s):
return hashlib.md5(s.encode('utf-8')).hexdigest()
else:
raise Error("Param should be string")
def home_dir():
if 'HOME' in os.environ:
return os.path.join(os.environ['HOME'], ".cdist")
else:
return None

7
cdist/argparse.py
View file

@ -116,6 +116,13 @@ def get_parsers():
# Config
parser['config_main'] = argparse.ArgumentParser(add_help=False)
parser['config_main'].add_argument(
'-C', '--cache-path-pattern',
help=('Specify custom cache path pattern. It can also be set '
'by CDIST_CACHE_PATH_PATTERN environment variable. If '
'it is not set then default hostdir is used.'),
dest='cache_path_pattern',
default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
parser['config_main'].add_argument(
'-c', '--conf-dir',
help=('Add configuration directory (can be repeated, '

6
cdist/config.py
View file

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
#
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
@ -223,7 +224,8 @@ class Config(object):
base_root_path=host_base_path,
host_dir_name=host_dir_name,
initial_manifest=args.manifest,
add_conf_dirs=args.conf_dir)
add_conf_dirs=args.conf_dir,
cache_path_pattern=args.cache_path_pattern)
remote = cdist.exec.remote.Remote(
target_host=target_host,
@ -260,7 +262,7 @@ class Config(object):
self.manifest.run_initial_manifest(self.local.initial_manifest)
self.iterate_until_finished()
self.local.save_cache()
self.local.save_cache(start_time)
self.log.info("Finished successful run in %s seconds",
time.time() - start_time)

75
cdist/exec/local.py
View file

@ -2,7 +2,7 @@
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
# 2016 Darko Poljak (darko.poljak at gmail.com)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
@ -29,6 +29,8 @@ import subprocess
import shutil
import logging
import tempfile
import time
import datetime
import cdist
import cdist.message
@ -51,7 +53,8 @@ class Local(object):
host_dir_name,
exec_path=sys.argv[0],
initial_manifest=None,
add_conf_dirs=None):
add_conf_dirs=None,
cache_path_pattern=None):
self.target_host = target_host
self.hostdir = host_dir_name
@ -60,6 +63,7 @@ class Local(object):
self.exec_path = exec_path
self.custom_initial_manifest = initial_manifest
self._add_conf_dirs = add_conf_dirs
self.cache_path_pattern = cache_path_pattern
self._init_log()
self._init_permissions()
@ -77,10 +81,7 @@ class Local(object):
@property
def home_dir(self):
if 'HOME' in os.environ:
return os.path.join(os.environ['HOME'], ".cdist")
else:
return None
return cdist.home_dir()
def _init_log(self):
self.log = logging.getLogger(self.target_host[0])
@ -239,7 +240,7 @@ class Local(object):
"""
if os.access(script, os.X_OK):
self.log.debug('%s is executable, running it', script)
command=[script]
command = [script]
else:
command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
self.log.debug('%s is NOT executable, running it with %s',
@ -249,18 +250,60 @@ class Local(object):
return self.run(command=command, env=env, return_output=return_output,
message_prefix=message_prefix, save_output=save_output)
def save_cache(self):
destination = os.path.join(self.cache_path, self.hostdir)
def _cache_subpath_repl(self, matchobj):
if matchobj.group(2) == '%P':
repl = str(os.getpid())
elif matchobj.group(2) == '%h':
repl = self.hostdir
elif matchobj.group(2) == '%N':
repl = self.target_host[0]
return matchobj.group(1) + repl
def _cache_subpath(self, start_time=time.time(), path_format=None):
if path_format:
repl_func = self._cache_subpath_repl
cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func,
path_format)
dt = datetime.datetime.fromtimestamp(start_time)
cache_subpath = dt.strftime(cache_subpath)
else:
cache_subpath = self.hostdir
i = 0
while i < len(cache_subpath) and cache_subpath[i] == os.sep:
i += 1
cache_subpath = cache_subpath[i:]
if not cache_subpath:
cache_subpath = self.hostdir
return cache_subpath
def save_cache(self, start_time=time.time()):
self.log.debug("cache subpath pattern: {}".format(
self.cache_path_pattern))
cache_subpath = self._cache_subpath(start_time,
self.cache_path_pattern)
self.log.debug("cache subpath: {}".format(cache_subpath))
destination = os.path.join(self.cache_path, cache_subpath)
self.log.debug("Saving " + self.base_path + " to " + destination)
try:
if os.path.exists(destination):
shutil.rmtree(destination)
except PermissionError as e:
raise cdist.Error(
"Cannot delete old cache %s: %s" % (destination, e))
if not os.path.exists(destination):
shutil.move(self.base_path, destination)
else:
for direntry in os.listdir(self.base_path):
srcentry = os.path.join(self.base_path, direntry)
destentry = os.path.join(destination, direntry)
try:
if os.path.isdir(destentry):
shutil.rmtree(destentry)
elif os.path.exists(destentry):
os.remove(destentry)
except (PermissionError, OSError) as e:
raise cdist.Error(
"Cannot delete old cache entry {}: {}".format(
destentry, e))
shutil.move(srcentry, destentry)
# add target_host since cache dir can be hash-ed target_host
host_cache_path = os.path.join(destination, "target_host")
with open(host_cache_path, 'w') as hostf:

38
cdist/hostsource.py
View file

@ -1,6 +1,6 @@
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at gmail.com)
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
#
# This file is part of cdist.
#
@ -22,6 +22,25 @@
import fileinput
def hostfile_process_line(line, strip_func=str.strip):
"""Return entry from read line or None if no entry present."""
if not line:
return None
# remove comment if present
comment_index = line.find('#')
if comment_index >= 0:
foo = line[:comment_index]
else:
foo = line
# remove leading and trailing whitespaces
foo = strip_func(foo)
# skip empty lines
if foo:
return foo
else:
return None
class HostSource(object):
"""
Host source object.
@ -32,22 +51,7 @@ class HostSource(object):
self.source = source
def _process_file_line(self, line):
"""Return host from read line or None if no host present."""
if not line:
return None
# remove comment if present
comment_index = line.find('#')
if comment_index >= 0:
host = line[:comment_index]
else:
host = line
# remove leading and trailing whitespaces
host = host.strip()
# skip empty lines
if host:
return host
else:
return None
return hostfile_process_line(line)
def _hosts_from_sequence(self):
for host in self.source:

37
cdist/test/exec/local.py
View file

@ -26,6 +26,8 @@ import getpass
import shutil
import string
import random
import time
import datetime
import cdist
from cdist import test
@ -224,6 +226,41 @@ class LocalTestCase(test.CdistTestCase):
self.assertTrue(os.path.isdir(self.local.bin_path))
self.assertTrue(os.path.isdir(self.local.conf_path))
def test_cache_subpath(self):
start_time = time.time()
dt = datetime.datetime.fromtimestamp(start_time)
pid = str(os.getpid())
cases = [
['', self.local.hostdir, ],
['/', self.local.hostdir, ],
['//', self.local.hostdir, ],
['/%%h', '%h', ],
['%%h', '%h', ],
['%P', pid, ],
['x%P', 'x' + pid, ],
['%h', self.hostdir, ],
['%h/%Y-%m-%d/%H%M%S%f%P',
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
['/%h/%Y-%m-%d/%H%M%S%f%P',
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
['%Y-%m-%d/%H%M%S%f%P/%h',
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
['///%Y-%m-%d/%H%M%S%f%P/%h',
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
['%h/%Y-%m-%d/%H%M%S-%P',
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ],
['%Y-%m-%d/%H%M%S-%P/%h',
dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ],
['%N', self.local.target_host[0], ],
]
for x in cases:
x.append(self.local._cache_subpath(start_time, x[0]))
# for fmt, expected, actual in cases:
# print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual))
for fmt, expected, actual in cases:
self.assertEqual(expected, actual)
if __name__ == "__main__":
import unittest

2
completions/bash/cdist-completion.bash
View file

@ -37,7 +37,7 @@ _cdist()
;;
config|install)
opts="-h --help -d --debug -v --verbose -b --beta \
-c --conf-dir -f --file -i --initial-manifest -j --jobs \
-C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs \
-n --dry-run -o --out-dir -p --parallel -s --sequential \
--remote-copy --remote-exec"
COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )

2
completions/zsh/_cdist
View file

@ -36,7 +36,7 @@ _cdist()
esac
;;
config|install)
opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
opts=(-h --help -d --debug -v --verbose -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec)
compadd "$@" -- $opts
;;
*)

3
docs/src/cdist-reference.rst.sh
View file

@ -276,4 +276,7 @@ CDIST_REMOTE_COPY
CDIST_BETA
Enable beta functionalities.
CDIST_CACHE_PATH_PATTERN
Custom cache path pattern.
eof

40
docs/src/man1/cdist.rst
View file

@ -15,13 +15,13 @@ SYNOPSIS
cdist banner [-h] [-d] [-v]
cdist config [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s]
cdist config [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[host [host ...]]
cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE]
[-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s]
cdist install [-h] [-d] [-v] [-b] [-C CACHE_PATH_PATTERN] [-c CONF_DIR]
[-f HOSTFILE] [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH]
[--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC]
[host [host ...]]
@ -76,6 +76,13 @@ Configure/install one or more hosts.
Can also be enabled using CDIST_BETA env var.
.. option:: -C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN
Sepcify custom cache path pattern. It can also be set by
CDIST_CACHE_PATH_PATTERN environment variable. If it is not set then
default hostdir is used. For more info on format see
:strong:`CACHE PATH PATTERN FORMAT` below.
.. option:: -c CONF_DIR, --conf-dir CONF_DIR
Add a configuration directory. Can be specified multiple times.
@ -91,7 +98,8 @@ Configure/install one or more hosts.
Read specified file for a list of additional hosts to operate on
or if '-' is given, read stdin (one host per line).
If no host or host file is specified then, by default,
read hosts from stdin. For the file format see below.
read hosts from stdin. For the file format see
:strong:`HOSTFILE FORMAT` below.
.. option:: -i MANIFEST, --initial-manifest MANIFEST
@ -145,6 +153,24 @@ removed. Then all leading and trailing whitespace characters are stripped.
If such a line results in empty line it is ignored/skipped. Otherwise,
host string is used.
CACHE PATH PATTERN FORMAT
~~~~~~~~~~~~~~~~~~~~~~~~~
Cache path pattern specifies path for a cache directory subdirectory.
In the path, '%N' will be substituted by the target host, '%h' will
be substituted by the calculated host directory, '%P' will be substituted
by the current process id. All format codes that
:strong:`python` :strong:`datetime.strftime()` function supports, except
'%h', are supported. These date/time directives format cdist config/install
start time.
If empty pattern is specified then default calculated host directory
is used.
Calculated host directory is a hash of a host cdist operates on.
Resulting path is used to specify cache path subdirectory under which
current host cache data are saved.
SHELL
-----
@ -247,6 +273,9 @@ CDIST_REMOTE_COPY
CDIST_BETA
Enable beta functionality.
CDIST_CACHE_PATH_PATTERN
Custom cache path pattern.
EXIT STATUS
-----------
The following exit values shall be returned:
@ -261,6 +290,7 @@ AUTHORS
Originally written by Nico Schottelius <nico-cdist--@--schottelius.org>
and Steven Armstrong <steven-cdist--@--armstrong.cc>.
CAVEATS
-------
When operating in parallel, either by operating in parallel for each host