tom/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'enhance_cache'

Browse source 
Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	cdist/test/code/__init__.py
	cdist/test/config_install/__init__.py
	cdist/test/explorer/__init__.py
This commit is contained in:
Nico Schottelius 2013-08-28 15:43:03 +02:00
commit e6feee14fb
12 changed files with 87 additions and 67 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cdist/config_install.py
View file

@ -117,7 +117,6 @@ class ConfigInstall(object):
try:
local = cdist.exec.local.Local(
target_host=host,
exec_path=sys.argv[0],
initial_manifest=args.manifest,
out_path=args.out_path,
add_conf_dirs=args.conf_dir)

2
cdist/core/code.py
View file

@ -89,7 +89,7 @@ class Code(object):
self.remote = remote
self.env = {
'__target_host': self.target_host,
'__global': self.local.out_path,
'__global': self.local.base_path,
}
def _run_gencode(self, cdist_object, which):

2
cdist/core/manifest.py
View file

@ -94,7 +94,7 @@ class Manifest(object):
self.env = {
'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
'__cdist_type_base_path': self.local.type_path, # for use in type emulator
'__global': self.local.out_path,
'__global': self.local.base_path,
'__target_host': self.target_host,
}
if self.log.getEffectiveLevel() == logging.DEBUG:

23
cdist/emulator.py
View file

@ -28,6 +28,15 @@ import sys
import cdist
from cdist import core
class MissingRequiredEnvironmentVariableError(cdist.Error):
def __init__(self, name):
self.name = name
self.message = "Emulator requires the environment variable %s to be setup" % self.name
def __str__(self):
return self.message
class Emulator(object):
def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
self.argv = argv
@ -36,12 +45,16 @@ class Emulator(object):
self.object_id = ''
self.global_path = self.env['__global']
self.target_host = self.env['__target_host']
try:
self.global_path = self.env['__global']
self.target_host = self.env['__target_host']
# Internally only
self.object_source = self.env['__cdist_manifest']
self.type_base_path = self.env['__cdist_type_base_path']
# Internally only
self.object_source = self.env['__cdist_manifest']
self.type_base_path = self.env['__cdist_type_base_path']
except KeyError as e:
raise MissingRequiredEnvironmentVariableError(e.args[0])
self.object_base_path = os.path.join(self.global_path, "object")

24
cdist/exec/local.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -41,18 +41,18 @@ class Local(object):
"""
def __init__(self,
target_host,
exec_path,
exec_path=sys.argv[0],
initial_manifest=None,
out_path=None,
base_path=None,
add_conf_dirs=None):
self.target_host = target_host
# FIXME: stopped: create base that does not require moving later
if out_path:
self.out_path = out_path
if base_path:
self.base_path = base_path
else:
self.out_path = tempfile.mkdtemp()
self.base_path = tempfile.mkdtemp()
# FIXME: as well
self._init_cache_dir(None)
@ -88,10 +88,10 @@ class Local(object):
def _init_paths(self):
# Depending on out_path
self.bin_path = os.path.join(self.out_path, "bin")
self.conf_path = os.path.join(self.out_path, "conf")
self.global_explorer_out_path = os.path.join(self.out_path, "explorer")
self.object_path = os.path.join(self.out_path, "object")
self.bin_path = os.path.join(self.base_path, "bin")
self.conf_path = os.path.join(self.base_path, "conf")
self.global_explorer_out_path = os.path.join(self.base_path, "explorer")
self.object_path = os.path.join(self.base_path, "object")
# Depending on conf_path
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
@ -185,10 +185,10 @@ class Local(object):
def save_cache(self):
destination = os.path.join(self.cache_path, self.target_host)
self.log.debug("Saving " + self.out_path + " to " + destination)
self.log.debug("Saving " + self.base_path + " to " + destination)
if os.path.exists(destination):
shutil.rmtree(destination)
shutil.move(self.out_path, destination)
shutil.move(self.base_path, destination)
def _create_conf_path_and_link_conf_dirs(self):
# Link destination directories

14
cdist/exec/remote.py
View file

@ -43,12 +43,20 @@ class Remote(object):
Directly accessing the remote side from python code is a bug.
"""
def __init__(self, target_host, remote_exec, remote_copy):
def __init__(self,
target_host,
remote_exec,
remote_copy,
base_path=None):
self.target_host = target_host
self.base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist")
self._exec = remote_exec
self._copy = remote_copy
if base_path:
self.base_path = base_path
else:
self.base_path = "/var/lib/cdist"
self.conf_path = os.path.join(self.base_path, "conf")
self.object_path = os.path.join(self.base_path, "object")
@ -141,6 +149,6 @@ class Remote(object):
except subprocess.CalledProcessError:
raise cdist.Error("Command failed: " + " ".join(command))
except OSError as error:
raise cdist.Error(" ".join(*args) + ": " + error.args[1])
raise cdist.Error(" ".join(command) + ": " + error.args[1])
except UnicodeDecodeError:
raise DecodeError(command)

18
cdist/test/code/__init__.py
View file

@ -39,18 +39,16 @@ conf_dir = op.join(fixtures, 'conf')
class CodeTestCase(test.CdistTestCase):
def setUp(self):
self.target_host = 'localhost'
self.out_path = self.mkdtemp()
self.local_dir = self.mkdtemp()
self.local = local.Local(
target_host=self.target_host,
out_path = self.out_path,
base_path = self.local_dir,
exec_path = cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
self.remote_base_path = self.mkdtemp()
self.remote_dir = self.mkdtemp()
remote_exec = self.remote_exec
remote_copy = self.remote_copy
self.remote = remote.Remote(
@ -58,7 +56,7 @@ class CodeTestCase(test.CdistTestCase):
remote_exec=remote_exec,
remote_copy=remote_copy)
self.remote.base_path = self.remote_base_path
self.remote.create_files_dirs()
tself.remote.create_files_dirs()
self.code = code.Code(self.target_host, self.local, self.remote)
@ -67,8 +65,8 @@ class CodeTestCase(test.CdistTestCase):
self.cdist_object.create()
def tearDown(self):
shutil.rmtree(self.out_path)
shutil.rmtree(self.remote_base_path)
shutil.rmtree(self.local_dir)
shutil.rmtree(self.remote_dir)
def test_run_gencode_local_environment(self):
output_string = self.code.run_gencode_local(self.cdist_object)
@ -79,7 +77,7 @@ class CodeTestCase(test.CdistTestCase):
key = junk.split(' ')[1]
output_dict[key] = value
self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path)
self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
@ -94,7 +92,7 @@ class CodeTestCase(test.CdistTestCase):
key = junk.split(' ')[1]
output_dict[key] = value
self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path)
self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)

20
cdist/test/config_install/__init__.py
View file

@ -47,19 +47,19 @@ class ConfigInstallRunTestCase(test.CdistTestCase):
os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp()
self.out_dir = os.path.join(self.temp_dir, "out")
self.remote_out_dir = os.path.join(self.temp_dir, "remote")
os.environ['__cdist_out_dir'] = self.out_dir
os.environ['__cdist_remote_out_dir'] = self.remote_out_dir
self.local = cdist.exec.local(
self.local_dir = os.path.join(self.temp_dir, "local")
os.mkdir(self.local_dir)
self.local = cdist.exec.local.Local(
target_host=self.target_host,
exec_path=test.cdist_exec_path)
self.remote = cdist.exec.remote(
base_path=self.local_dir)
self.remote_dir = os.path.join(self.temp_dir, "remote")
os.mkdir(self.remote_dir)
self.remote = cdist.exec.remote.Remote(
target_host=self.target_host,
remote_copy=self.remote_copy,
remote_exec=self.remote_exec)
remote_exec=self.remote_exec,
base_path=self.remote_dir)
self.local.object_path = object_base_path
self.local.type_path = type_base_path

19
cdist/test/emulator/__init__.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -45,11 +45,11 @@ class EmulatorTestCase(test.CdistTestCase):
self.temp_dir = self.mkdtemp()
handle, self.script = self.mkstemp(dir=self.temp_dir)
os.close(handle)
out_path = self.temp_dir
base_path = self.temp_dir
self.local = local.Local(
target_host=self.target_host,
out_path=out_path,
base_path=base_path,
exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
@ -107,11 +107,11 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
def setUp(self):
self.temp_dir = self.mkdtemp()
out_path = os.path.join(self.temp_dir, "out")
base_path = os.path.join(self.temp_dir, "out")
self.local = local.Local(
target_host=self.target_host,
out_path=out_path,
base_path=base_path,
exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
@ -134,13 +134,13 @@ class ArgumentsTestCase(test.CdistTestCase):
def setUp(self):
self.temp_dir = self.mkdtemp()
out_path = self.temp_dir
base_path = self.temp_dir
handle, self.script = self.mkstemp(dir=self.temp_dir)
os.close(handle)
self.local = local.Local(
target_host=self.target_host,
out_path=out_path,
base_path=base_path,
exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
@ -182,6 +182,7 @@ class ArgumentsTestCase(test.CdistTestCase):
object_id = 'some-id'
value = 'some value'
argv = [type_name, object_id, '--required1', value, '--required2', value]
print(self.env)
os.environ.update(self.env)
emu = emulator.Emulator(argv)
emu.run()
@ -226,11 +227,11 @@ class StdinTestCase(test.CdistTestCase):
os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp()
out_path = os.path.join(self.temp_dir, "out")
base_path = os.path.join(self.temp_dir, "out")
self.local = local.Local(
target_host=self.target_host,
out_path=out_path,
base_path=base_path,
exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir])

18
cdist/test/explorer/__init__.py
View file

@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
#
# This file is part of cdist.
#
@ -39,25 +39,25 @@ conf_dir = op.join(fixtures, "conf")
class ExplorerClassTestCase(test.CdistTestCase):
def setUp(self):
self.target_host = 'localhost'
self.temp_dir = self.mkdtemp()
self.out_path = os.path.join(self.temp_dir, "out")
self.local_path = os.path.join(self.temp_dir, "local")
self.remote_base_path = os.path.join(self.temp_dir, "remote")
os.makedirs(self.remote_base_path)
self.local = local.Local(
target_host=self.target_host,
out_path=self.out_path,
base_path=self.local_path,
exec_path=test.cdist_exec_path,
add_conf_dirs=[conf_dir])
add_conf_dirs=[conf_dir],
)
self.local.create_files_dirs()
self.remote = remote.Remote(
self.target_host,
self.remote_exec,
self.remote_copy)
target_host=self.target_host,
remote_exec=self.remote_exec,
remote_copy=self.remote_copy,
base_path=self.remote_base_path)
self.remote.create_files_dirs()
self.explorer = explorer.Explorer(

8
cdist/test/manifest/__init__.py
View file

@ -46,11 +46,11 @@ class ManifestTestCase(test.CdistTestCase):
self.orig_environ = os.environ
os.environ = os.environ.copy()
self.temp_dir = self.mkdtemp()
self.target_host = 'localhost'
out_path = self.temp_dir
self.local = local.Local(
target_host=self.target_host,
out_path=out_path,
base_path=out_path,
exec_path = cdist.test.cdist_exec_path,
add_conf_dirs=[conf_dir])
self.local.create_files_dirs()
@ -78,7 +78,7 @@ class ManifestTestCase(test.CdistTestCase):
output_dict[key] = value
self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path)
self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path)
self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
@ -99,7 +99,7 @@ class ManifestTestCase(test.CdistTestCase):
output_dict[key] = value
self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
self.assertEqual(output_dict['__target_host'], self.local.target_host)
self.assertEqual(output_dict['__global'], self.local.out_path)
self.assertEqual(output_dict['__global'], self.local.base_path)
self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path)
self.assertEqual(output_dict['__type'], cdist_type.absolute_path)
self.assertEqual(output_dict['__object'], cdist_object.absolute_path)

5
scripts/cdist
View file

@ -65,8 +65,9 @@ def commandline():
dest='manifest', required=False)
parser['configinstall'].add_argument('-n', '--dry-run',
help='Do not execute code', action='store_true')
parser['configinstall'].add_argument('-o', '--out-path',
help='Directory prefix to save cdist output in')
parser['configinstall'].add_argument('-o', '--out-dir',
help='Directory to save cdist output in',
destination="out_path")
parser['configinstall'].add_argument('-p', '--parallel',
help='Operate on multiple hosts in parallel',
action='store_true', dest='parallel')