Merge custom cache path pattern from beta branch.
This commit is contained in:
Darko Poljak
parent
b7873abf07
commit
2a9bd77550
10 changed files with 176 additions and 43 deletions
|
|
@ -114,3 +114,10 @@ def str_hash(s):
|
|||
return hashlib.md5(s.encode('utf-8')).hexdigest()
|
||||
else:
|
||||
raise Error("Param should be string")
|
||||
|
||||
|
||||
def home_dir():
|
||||
if 'HOME' in os.environ:
|
||||
return os.path.join(os.environ['HOME'], ".cdist")
|
||||
else:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -116,6 +116,13 @@ def get_parsers():
|
|||
|
||||
# Config
|
||||
parser['config_main'] = argparse.ArgumentParser(add_help=False)
|
||||
parser['config_main'].add_argument(
|
||||
'-C', '--cache-path-pattern',
|
||||
help=('Specify custom cache path pattern. It can also be set '
|
||||
'by CDIST_CACHE_PATH_PATTERN environment variable. If '
|
||||
'it is not set then default hostdir is used.'),
|
||||
dest='cache_path_pattern',
|
||||
default=os.environ.get('CDIST_CACHE_PATH_PATTERN'))
|
||||
parser['config_main'].add_argument(
|
||||
'-c', '--conf-dir',
|
||||
help=('Add configuration directory (can be repeated, '
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org)
|
||||
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
|
|
@ -223,7 +224,8 @@ class Config(object):
|
|||
base_root_path=host_base_path,
|
||||
host_dir_name=host_dir_name,
|
||||
initial_manifest=args.manifest,
|
||||
add_conf_dirs=args.conf_dir)
|
||||
add_conf_dirs=args.conf_dir,
|
||||
cache_path_pattern=args.cache_path_pattern)
|
||||
|
||||
remote = cdist.exec.remote.Remote(
|
||||
target_host=target_host,
|
||||
|
|
@ -260,7 +262,7 @@ class Config(object):
|
|||
self.manifest.run_initial_manifest(self.local.initial_manifest)
|
||||
self.iterate_until_finished()
|
||||
|
||||
self.local.save_cache()
|
||||
self.local.save_cache(start_time)
|
||||
self.log.info("Finished successful run in %s seconds",
|
||||
time.time() - start_time)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
#
|
||||
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
|
||||
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
|
|
@ -29,6 +29,8 @@ import subprocess
|
|||
import shutil
|
||||
import logging
|
||||
import tempfile
|
||||
import time
|
||||
import datetime
|
||||
|
||||
import cdist
|
||||
import cdist.message
|
||||
|
|
@ -51,7 +53,8 @@ class Local(object):
|
|||
host_dir_name,
|
||||
exec_path=sys.argv[0],
|
||||
initial_manifest=None,
|
||||
add_conf_dirs=None):
|
||||
add_conf_dirs=None,
|
||||
cache_path_pattern=None):
|
||||
|
||||
self.target_host = target_host
|
||||
self.hostdir = host_dir_name
|
||||
|
|
@ -60,6 +63,7 @@ class Local(object):
|
|||
self.exec_path = exec_path
|
||||
self.custom_initial_manifest = initial_manifest
|
||||
self._add_conf_dirs = add_conf_dirs
|
||||
self.cache_path_pattern = cache_path_pattern
|
||||
|
||||
self._init_log()
|
||||
self._init_permissions()
|
||||
|
|
@ -77,10 +81,7 @@ class Local(object):
|
|||
|
||||
@property
|
||||
def home_dir(self):
|
||||
if 'HOME' in os.environ:
|
||||
return os.path.join(os.environ['HOME'], ".cdist")
|
||||
else:
|
||||
return None
|
||||
return cdist.home_dir()
|
||||
|
||||
def _init_log(self):
|
||||
self.log = logging.getLogger(self.target_host[0])
|
||||
|
|
@ -239,28 +240,70 @@ class Local(object):
|
|||
"""
|
||||
if os.access(script, os.X_OK):
|
||||
self.log.debug('%s is executable, running it', script)
|
||||
command=[script]
|
||||
command = [script]
|
||||
else:
|
||||
command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"]
|
||||
self.log.debug('%s is NOT executable, running it with %s',
|
||||
script, " ".join(command))
|
||||
script, " ".join(command))
|
||||
command.append(script)
|
||||
|
||||
return self.run(command=command, env=env, return_output=return_output,
|
||||
message_prefix=message_prefix, save_output=save_output)
|
||||
|
||||
def save_cache(self):
|
||||
destination = os.path.join(self.cache_path, self.hostdir)
|
||||
def _cache_subpath_repl(self, matchobj):
|
||||
if matchobj.group(2) == '%P':
|
||||
repl = str(os.getpid())
|
||||
elif matchobj.group(2) == '%h':
|
||||
repl = self.hostdir
|
||||
elif matchobj.group(2) == '%N':
|
||||
repl = self.target_host[0]
|
||||
|
||||
return matchobj.group(1) + repl
|
||||
|
||||
def _cache_subpath(self, start_time=time.time(), path_format=None):
|
||||
if path_format:
|
||||
repl_func = self._cache_subpath_repl
|
||||
cache_subpath = re.sub(r'([^%]|^)(%h|%P|%N)', repl_func,
|
||||
path_format)
|
||||
dt = datetime.datetime.fromtimestamp(start_time)
|
||||
cache_subpath = dt.strftime(cache_subpath)
|
||||
else:
|
||||
cache_subpath = self.hostdir
|
||||
|
||||
i = 0
|
||||
while i < len(cache_subpath) and cache_subpath[i] == os.sep:
|
||||
i += 1
|
||||
cache_subpath = cache_subpath[i:]
|
||||
if not cache_subpath:
|
||||
cache_subpath = self.hostdir
|
||||
return cache_subpath
|
||||
|
||||
def save_cache(self, start_time=time.time()):
|
||||
self.log.debug("cache subpath pattern: {}".format(
|
||||
self.cache_path_pattern))
|
||||
cache_subpath = self._cache_subpath(start_time,
|
||||
self.cache_path_pattern)
|
||||
self.log.debug("cache subpath: {}".format(cache_subpath))
|
||||
destination = os.path.join(self.cache_path, cache_subpath)
|
||||
self.log.debug("Saving " + self.base_path + " to " + destination)
|
||||
|
||||
try:
|
||||
if os.path.exists(destination):
|
||||
shutil.rmtree(destination)
|
||||
except PermissionError as e:
|
||||
raise cdist.Error(
|
||||
"Cannot delete old cache %s: %s" % (destination, e))
|
||||
if not os.path.exists(destination):
|
||||
shutil.move(self.base_path, destination)
|
||||
else:
|
||||
for direntry in os.listdir(self.base_path):
|
||||
srcentry = os.path.join(self.base_path, direntry)
|
||||
destentry = os.path.join(destination, direntry)
|
||||
try:
|
||||
if os.path.isdir(destentry):
|
||||
shutil.rmtree(destentry)
|
||||
elif os.path.exists(destentry):
|
||||
os.remove(destentry)
|
||||
except (PermissionError, OSError) as e:
|
||||
raise cdist.Error(
|
||||
"Cannot delete old cache entry {}: {}".format(
|
||||
destentry, e))
|
||||
shutil.move(srcentry, destentry)
|
||||
|
||||
shutil.move(self.base_path, destination)
|
||||
# add target_host since cache dir can be hash-ed target_host
|
||||
host_cache_path = os.path.join(destination, "target_host")
|
||||
with open(host_cache_path, 'w') as hostf:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
#
|
||||
# 2016 Darko Poljak (darko.poljak at gmail.com)
|
||||
# 2016-2017 Darko Poljak (darko.poljak at gmail.com)
|
||||
#
|
||||
# This file is part of cdist.
|
||||
#
|
||||
|
|
@ -22,6 +22,25 @@
|
|||
import fileinput
|
||||
|
||||
|
||||
def hostfile_process_line(line, strip_func=str.strip):
|
||||
"""Return entry from read line or None if no entry present."""
|
||||
if not line:
|
||||
return None
|
||||
# remove comment if present
|
||||
comment_index = line.find('#')
|
||||
if comment_index >= 0:
|
||||
foo = line[:comment_index]
|
||||
else:
|
||||
foo = line
|
||||
# remove leading and trailing whitespaces
|
||||
foo = strip_func(foo)
|
||||
# skip empty lines
|
||||
if foo:
|
||||
return foo
|
||||
else:
|
||||
return None
|
||||
|
||||
|
||||
class HostSource(object):
|
||||
"""
|
||||
Host source object.
|
||||
|
|
@ -32,22 +51,7 @@ class HostSource(object):
|
|||
self.source = source
|
||||
|
||||
def _process_file_line(self, line):
|
||||
"""Return host from read line or None if no host present."""
|
||||
if not line:
|
||||
return None
|
||||
# remove comment if present
|
||||
comment_index = line.find('#')
|
||||
if comment_index >= 0:
|
||||
host = line[:comment_index]
|
||||
else:
|
||||
host = line
|
||||
# remove leading and trailing whitespaces
|
||||
host = host.strip()
|
||||
# skip empty lines
|
||||
if host:
|
||||
return host
|
||||
else:
|
||||
return None
|
||||
return hostfile_process_line(line)
|
||||
|
||||
def _hosts_from_sequence(self):
|
||||
for host in self.source:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ import getpass
|
|||
import shutil
|
||||
import string
|
||||
import random
|
||||
import time
|
||||
import datetime
|
||||
|
||||
import cdist
|
||||
from cdist import test
|
||||
|
|
@ -224,6 +226,41 @@ class LocalTestCase(test.CdistTestCase):
|
|||
self.assertTrue(os.path.isdir(self.local.bin_path))
|
||||
self.assertTrue(os.path.isdir(self.local.conf_path))
|
||||
|
||||
def test_cache_subpath(self):
|
||||
start_time = time.time()
|
||||
dt = datetime.datetime.fromtimestamp(start_time)
|
||||
pid = str(os.getpid())
|
||||
cases = [
|
||||
['', self.local.hostdir, ],
|
||||
['/', self.local.hostdir, ],
|
||||
['//', self.local.hostdir, ],
|
||||
['/%%h', '%h', ],
|
||||
['%%h', '%h', ],
|
||||
['%P', pid, ],
|
||||
['x%P', 'x' + pid, ],
|
||||
['%h', self.hostdir, ],
|
||||
['%h/%Y-%m-%d/%H%M%S%f%P',
|
||||
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
|
||||
['/%h/%Y-%m-%d/%H%M%S%f%P',
|
||||
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ],
|
||||
['%Y-%m-%d/%H%M%S%f%P/%h',
|
||||
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
|
||||
['///%Y-%m-%d/%H%M%S%f%P/%h',
|
||||
dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ],
|
||||
['%h/%Y-%m-%d/%H%M%S-%P',
|
||||
dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ],
|
||||
['%Y-%m-%d/%H%M%S-%P/%h',
|
||||
dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ],
|
||||
['%N', self.local.target_host[0], ],
|
||||
]
|
||||
for x in cases:
|
||||
x.append(self.local._cache_subpath(start_time, x[0]))
|
||||
# for fmt, expected, actual in cases:
|
||||
# print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual))
|
||||
for fmt, expected, actual in cases:
|
||||
self.assertEqual(expected, actual)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import unittest
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue