tom/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'enhance_cache'

Browse source 
Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	cdist/test/code/__init__.py
	cdist/test/config_install/__init__.py
	cdist/test/explorer/__init__.py
This commit is contained in:
Nico Schottelius 2013-08-28 15:43:03 +02:00
commit e6feee14fb
12 changed files with 87 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cdist/config_install.py
View file

@ -117,7 +117,6 @@ class ConfigInstall(object):
try: try:
local = cdist.exec.local.Local( local = cdist.exec.local.Local(
target_host=host, target_host=host,
exec_path=sys.argv[0],
initial_manifest=args.manifest, initial_manifest=args.manifest,
out_path=args.out_path, out_path=args.out_path,
add_conf_dirs=args.conf_dir) add_conf_dirs=args.conf_dir)

2
cdist/core/code.py
View file

@ -89,7 +89,7 @@ class Code(object):
self.remote = remote self.remote = remote
self.env = { self.env = {
'__target_host': self.target_host, '__target_host': self.target_host,
'__global': self.local.out_path, '__global': self.local.base_path,
} }
def _run_gencode(self, cdist_object, which): def _run_gencode(self, cdist_object, which):

2
cdist/core/manifest.py
View file

@ -94,7 +94,7 @@ class Manifest(object):
self.env = { self.env = {
'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
'__cdist_type_base_path': self.local.type_path, # for use in type emulator '__cdist_type_base_path': self.local.type_path, # for use in type emulator
'__global': self.local.out_path, '__global': self.local.base_path,
'__target_host': self.target_host, '__target_host': self.target_host,
} }
if self.log.getEffectiveLevel() == logging.DEBUG: if self.log.getEffectiveLevel() == logging.DEBUG:

13
cdist/emulator.py
View file

@ -28,6 +28,15 @@ import sys
import cdist import cdist
from cdist import core from cdist import core
class MissingRequiredEnvironmentVariableError(cdist.Error):
def __init__(self, name):
self.name = name
self.message = "Emulator requires the environment variable %s to be setup" % self.name
def __str__(self):
return self.message
class Emulator(object): class Emulator(object):
def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
self.argv = argv self.argv = argv
@ -36,6 +45,7 @@ class Emulator(object):
self.object_id = '' self.object_id = ''
try:
self.global_path = self.env['__global'] self.global_path = self.env['__global']
self.target_host = self.env['__target_host'] self.target_host = self.env['__target_host']
@ -43,6 +53,9 @@ class Emulator(object):
self.object_source = self.env['__cdist_manifest'] self.object_source = self.env['__cdist_manifest']
self.type_base_path = self.env['__cdist_type_base_path'] self.type_base_path = self.env['__cdist_type_base_path']
except KeyError as e:
raise MissingRequiredEnvironmentVariableError(e.args[0])
self.object_base_path = os.path.join(self.global_path, "object") self.object_base_path = os.path.join(self.global_path, "object")
self.type_name = os.path.basename(argv[0]) self.type_name = os.path.basename(argv[0])

24
cdist/exec/local.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -41,18 +41,18 @@ class Local(object):
""" """
def __init__(self, def __init__(self,
target_host, target_host,
exec_path, exec_path=sys.argv[0],
initial_manifest=None, initial_manifest=None,
out_path=None, base_path=None,
add_conf_dirs=None): add_conf_dirs=None):
self.target_host = target_host self.target_host = target_host
# FIXME: stopped: create base that does not require moving later # FIXME: stopped: create base that does not require moving later
if out_path: if base_path:
self.out_path = out_path self.base_path = base_path
else: else:
self.out_path = tempfile.mkdtemp() self.base_path = tempfile.mkdtemp()
# FIXME: as well # FIXME: as well
self._init_cache_dir(None) self._init_cache_dir(None)
@ -88,10 +88,10 @@ class Local(object):
def _init_paths(self): def _init_paths(self):
# Depending on out_path # Depending on out_path
self.bin_path = os.path.join(self.out_path, "bin") self.bin_path = os.path.join(self.base_path, "bin")
self.conf_path = os.path.join(self.out_path, "conf") self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_out_path = os.path.join(self.out_path, "explorer") self.global_explorer_out_path = os.path.join(self.base_path, "explorer")
self.object_path = os.path.join(self.out_path, "object") self.object_path = os.path.join(self.base_path, "object")
# Depending on conf_path # Depending on conf_path
self.global_explorer_path = os.path.join(self.conf_path, "explorer") self.global_explorer_path = os.path.join(self.conf_path, "explorer")
@ -185,10 +185,10 @@ class Local(object):
def save_cache(self): def save_cache(self):
destination = os.path.join(self.cache_path, self.target_host) destination = os.path.join(self.cache_path, self.target_host)
self.log.debug("Saving " + self.out_path + " to " + destination) self.log.debug("Saving " + self.base_path + " to " + destination)
if os.path.exists(destination): if os.path.exists(destination):
shutil.rmtree(destination) shutil.rmtree(destination)
shutil.move(self.out_path, destination) shutil.move(self.base_path, destination)
def _create_conf_path_and_link_conf_dirs(self): def _create_conf_path_and_link_conf_dirs(self):
# Link destination directories # Link destination directories

14
cdist/exec/remote.py
View file

@ -43,12 +43,20 @@ class Remote(object):
Directly accessing the remote side from python code is a bug. Directly accessing the remote side from python code is a bug.
""" """
def __init__(self, target_host, remote_exec, remote_copy): def __init__(self,
target_host,
remote_exec,
remote_copy,
base_path=None):
self.target_host = target_host self.target_host = target_host
self.base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist")
self._exec = remote_exec self._exec = remote_exec
self._copy = remote_copy self._copy = remote_copy
if base_path:
self.base_path = base_path
else:
self.base_path = "/var/lib/cdist"
self.conf_path = os.path.join(self.base_path, "conf") self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object") self.object_path = os.path.join(self.base_path, "object")
@ -141,6 +149,6 @@ class Remote(object):
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(command)) raise cdist.Error("Command failed: " + " ".join(command))
except OSError as error: except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1]) raise cdist.Error(" ".join(command) + ": " + error.args[1])
except UnicodeDecodeError: except UnicodeDecodeError:
raise DecodeError(command) raise DecodeError(command)

18
cdist/test/code/__init__.py
View file

@ -39,18 +39,16 @@ conf_dir = op.join(fixtures, 'conf')
class CodeTestCase(test.CdistTestCase): class CodeTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
self.target_host = 'localhost' self.local_dir = self.mkdtemp()
self.out_path = self.mkdtemp()
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path = self.out_path, base_path = self.local_dir,
exec_path = cdist.test.cdist_exec_path, exec_path = cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir])
self.local.create_files_dirs() self.local.create_files_dirs()
self.remote_base_path = self.mkdtemp() self.remote_dir = self.mkdtemp()
remote_exec = self.remote_exec remote_exec = self.remote_exec
remote_copy = self.remote_copy remote_copy = self.remote_copy
self.remote = remote.Remote( self.remote = remote.Remote(
@ -58,7 +56,7 @@ class CodeTestCase(test.CdistTestCase):
remote_exec=remote_exec, remote_exec=remote_exec,
remote_copy=remote_copy) remote_copy=remote_copy)
self.remote.base_path = self.remote_base_path self.remote.base_path = self.remote_base_path
self.remote.create_files_dirs() tself.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote) self.code = code.Code(self.target_host, self.local, self.remote)
@ -67,8 +65,8 @@ class CodeTestCase(test.CdistTestCase):
self.cdist_object.create() self.cdist_object.create()
def tearDown(self): def tearDown(self):
shutil.rmtree(self.out_path) shutil.rmtree(self.local_dir)
shutil.rmtree(self.remote_base_path) shutil.rmtree(self.remote_dir)
def test_run_gencode_local_environment(self): def test_run_gencode_local_environment(self):
output_string = self.code.run_gencode_local(self.cdist_object) output_string = self.code.run_gencode_local(self.cdist_object)
@ -79,7 +77,7 @@ class CodeTestCase(test.CdistTestCase):
key = junk.split(' ')[1] key = junk.split(' ')[1]
output_dict[key] = value output_dict[key] = value
self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path) self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
@ -94,7 +92,7 @@ class CodeTestCase(test.CdistTestCase):
key = junk.split(' ')[1] key = junk.split(' ')[1]
output_dict[key] = value output_dict[key] = value
self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path) self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)

20
cdist/test/config_install/__init__.py
View file

@ -47,19 +47,19 @@ class ConfigInstallRunTestCase(test.CdistTestCase):
os.environ = os.environ.copy() os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
self.out_dir = os.path.join(self.temp_dir, "out") self.local_dir = os.path.join(self.temp_dir, "local")
self.remote_out_dir = os.path.join(self.temp_dir, "remote") os.mkdir(self.local_dir)
self.local = cdist.exec.local.Local(
os.environ['__cdist_out_dir'] = self.out_dir
os.environ['__cdist_remote_out_dir'] = self.remote_out_dir
self.local = cdist.exec.local(
target_host=self.target_host, target_host=self.target_host,
exec_path=test.cdist_exec_path) base_path=self.local_dir)
self.remote = cdist.exec.remote(
self.remote_dir = os.path.join(self.temp_dir, "remote")
os.mkdir(self.remote_dir)
self.remote = cdist.exec.remote.Remote(
target_host=self.target_host, target_host=self.target_host,
remote_copy=self.remote_copy, remote_copy=self.remote_copy,
remote_exec=self.remote_exec) remote_exec=self.remote_exec,
base_path=self.remote_dir)
self.local.object_path = object_base_path self.local.object_path = object_base_path
self.local.type_path = type_base_path self.local.type_path = type_base_path

19
cdist/test/emulator/__init__.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2012-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -45,11 +45,11 @@ class EmulatorTestCase(test.CdistTestCase):
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
handle, self.script = self.mkstemp(dir=self.temp_dir) handle, self.script = self.mkstemp(dir=self.temp_dir)
os.close(handle) os.close(handle)
out_path = self.temp_dir base_path = self.temp_dir
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path=out_path, base_path=base_path,
exec_path=test.cdist_exec_path, exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir])
self.local.create_files_dirs() self.local.create_files_dirs()
@ -107,11 +107,11 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
out_path = os.path.join(self.temp_dir, "out") base_path = os.path.join(self.temp_dir, "out")
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path=out_path, base_path=base_path,
exec_path=test.cdist_exec_path, exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir])
self.local.create_files_dirs() self.local.create_files_dirs()
@ -134,13 +134,13 @@ class ArgumentsTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
out_path = self.temp_dir base_path = self.temp_dir
handle, self.script = self.mkstemp(dir=self.temp_dir) handle, self.script = self.mkstemp(dir=self.temp_dir)
os.close(handle) os.close(handle)
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path=out_path, base_path=base_path,
exec_path=test.cdist_exec_path, exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir])
self.local.create_files_dirs() self.local.create_files_dirs()
@ -182,6 +182,7 @@ class ArgumentsTestCase(test.CdistTestCase):
object_id = 'some-id' object_id = 'some-id'
value = 'some value' value = 'some value'
argv = [type_name, object_id, '--required1', value, '--required2', value] argv = [type_name, object_id, '--required1', value, '--required2', value]
print(self.env)
os.environ.update(self.env) os.environ.update(self.env)
emu = emulator.Emulator(argv) emu = emulator.Emulator(argv)
emu.run() emu.run()
@ -226,11 +227,11 @@ class StdinTestCase(test.CdistTestCase):
os.environ = os.environ.copy() os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
out_path = os.path.join(self.temp_dir, "out") base_path = os.path.join(self.temp_dir, "out")
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path=out_path, base_path=base_path,
exec_path=test.cdist_exec_path, exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir])

18
cdist/test/explorer/__init__.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
# #
# This file is part of cdist. # This file is part of cdist.
# #
@ -39,25 +39,25 @@ conf_dir = op.join(fixtures, "conf")
class ExplorerClassTestCase(test.CdistTestCase): class ExplorerClassTestCase(test.CdistTestCase):
def setUp(self): def setUp(self):
self.target_host = 'localhost'
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
self.out_path = os.path.join(self.temp_dir, "out") self.local_path = os.path.join(self.temp_dir, "local")
self.remote_base_path = os.path.join(self.temp_dir, "remote") self.remote_base_path = os.path.join(self.temp_dir, "remote")
os.makedirs(self.remote_base_path) os.makedirs(self.remote_base_path)
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path=self.out_path, base_path=self.local_path,
exec_path=test.cdist_exec_path, exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir],
)
self.local.create_files_dirs() self.local.create_files_dirs()
self.remote = remote.Remote( self.remote = remote.Remote(
self.target_host, target_host=self.target_host,
self.remote_exec, remote_exec=self.remote_exec,
self.remote_copy) remote_copy=self.remote_copy,
base_path=self.remote_base_path)
self.remote.create_files_dirs() self.remote.create_files_dirs()
self.explorer = explorer.Explorer( self.explorer = explorer.Explorer(

8
cdist/test/manifest/__init__.py
View file

@ -46,11 +46,11 @@ class ManifestTestCase(test.CdistTestCase):
self.orig_environ = os.environ self.orig_environ = os.environ
os.environ = os.environ.copy() os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp() self.temp_dir = self.mkdtemp()
self.target_host = 'localhost'
out_path = self.temp_dir out_path = self.temp_dir
self.local = local.Local( self.local = local.Local(
target_host=self.target_host, target_host=self.target_host,
out_path=out_path, base_path=out_path,
exec_path = cdist.test.cdist_exec_path, exec_path = cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir]) add_conf_dirs=[conf_dir])
self.local.create_files_dirs() self.local.create_files_dirs()
@ -78,7 +78,7 @@ class ManifestTestCase(test.CdistTestCase):
output_dict[key] = value output_dict[key] = value
self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path) self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path)
self.assertEqual(output_dict['__manifest'], self.local.manifest_path) self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
@ -99,7 +99,7 @@ class ManifestTestCase(test.CdistTestCase):
output_dict[key] = value output_dict[key] = value
self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path) self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path)
self.assertEqual(output_dict['__type'], cdist_type.absolute_path) self.assertEqual(output_dict['__type'], cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], cdist_object.absolute_path) self.assertEqual(output_dict['__object'], cdist_object.absolute_path)

5
scripts/cdist
View file

@ -65,8 +65,9 @@ def commandline():
dest='manifest', required=False) dest='manifest', required=False)
parser['configinstall'].add_argument('-n', '--dry-run', parser['configinstall'].add_argument('-n', '--dry-run',
help='Do not execute code', action='store_true') help='Do not execute code', action='store_true')
parser['configinstall'].add_argument('-o', '--out-path', parser['configinstall'].add_argument('-o', '--out-dir',
help='Directory prefix to save cdist output in') help='Directory to save cdist output in',
destination="out_path")
parser['configinstall'].add_argument('-p', '--parallel', parser['configinstall'].add_argument('-p', '--parallel',
help='Operate on multiple hosts in parallel', help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel') action='store_true', dest='parallel')