From 10b27e63cae79b5d1600bb434e00a03eee939765 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 11:21:36 +0200 Subject: [PATCH 1/9] rename out_path -> out_dir for consistency Signed-off-by: Nico Schottelius --- scripts/cdist | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/cdist b/scripts/cdist index bbf45d17..d9c48b12 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -65,8 +65,9 @@ def commandline(): dest='manifest', required=False) parser['configinstall'].add_argument('-n', '--dry-run', help='Do not execute code', action='store_true') - parser['configinstall'].add_argument('-o', '--out-path', - help='Directory prefix to save cdist output in') + parser['configinstall'].add_argument('-o', '--out-dir', + help='Directory to save cdist output in', + destination="out_path") parser['configinstall'].add_argument('-p', '--parallel', help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') From b3cf339d06f16be0c011cd387699a475da7ad47e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 11:21:53 +0200 Subject: [PATCH 2/9] fallback to sys.argv[0] by default Signed-off-by: Nico Schottelius --- cdist/exec/local.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index b3478cf9..6e4029f3 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -41,7 +41,7 @@ class Local(object): """ def __init__(self, target_host, - exec_path, + exec_path=sys.argv[0], initial_manifest=None, out_path=None, add_conf_dirs=None): From 8298bb0bf5b8e19ee1188fdef89f08e43f8c85ea Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 11:33:44 +0200 Subject: [PATCH 3/9] fix test cdist.test.config_install.ConfigInstallRunTestCase Signed-off-by: Nico Schottelius --- cdist/config_install.py | 1 - cdist/exec/remote.py | 12 ++++++++++-- cdist/test/config_install/__init__.py | 22 +++++++++++----------- 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/cdist/config_install.py b/cdist/config_install.py index c7f141dd..2db4c4af 100644 --- a/cdist/config_install.py +++ b/cdist/config_install.py @@ -117,7 +117,6 @@ class ConfigInstall(object): try: local = cdist.exec.local.Local( target_host=host, - exec_path=sys.argv[0], initial_manifest=args.manifest, out_path=args.out_path, add_conf_dirs=args.conf_dir) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 647474fa..01f028a9 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -43,12 +43,20 @@ class Remote(object): Directly accessing the remote side from python code is a bug. """ - def __init__(self, target_host, remote_exec, remote_copy): + def __init__(self, + target_host, + remote_exec, + remote_copy, + base_path=None): self.target_host = target_host - self.base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist") self._exec = remote_exec self._copy = remote_copy + if base_path: + self.base_path = base_path + else: + self.base_path = "/var/lib/cdist" + self.conf_path = os.path.join(self.base_path, "conf") self.object_path = os.path.join(self.base_path, "object") diff --git a/cdist/test/config_install/__init__.py b/cdist/test/config_install/__init__.py index b8daf4f8..4047abe4 100644 --- a/cdist/test/config_install/__init__.py +++ b/cdist/test/config_install/__init__.py @@ -27,7 +27,6 @@ from cdist import test from cdist import core import cdist -import cdist.context import cdist.config import cdist.core.cdist_type import cdist.core.cdist_object @@ -49,22 +48,23 @@ class ConfigInstallRunTestCase(test.CdistTestCase): self.temp_dir = self.mkdtemp() self.out_dir = os.path.join(self.temp_dir, "out") - self.remote_out_dir = os.path.join(self.temp_dir, "remote") + os.mkdir(self.out_dir) + self.local = cdist.exec.local.Local( + target_host=self.target_host, + out_path=self.out_dir) - os.environ['__cdist_out_dir'] = self.out_dir - os.environ['__cdist_remote_out_dir'] = self.remote_out_dir - - self.context = cdist.context.Context( + self.remote_dir = os.path.join(self.temp_dir, "remote") + os.mkdir(self.remote_dir) + self.remote = cdist.exec.remote.Remote( target_host=self.target_host, remote_copy=self.remote_copy, remote_exec=self.remote_exec, - exec_path=test.cdist_exec_path, - debug=True) + base_path=self.remote_dir) - self.context.local.object_path = object_base_path - self.context.local.type_path = type_base_path + self.local.object_path = object_base_path + self.local.type_path = type_base_path - self.config = cdist.config.Config(self.context) + self.config = cdist.config.Config(self.local, self.remote) self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) self.object_index = dict((o.name, o) for o in self.objects) From a76d8bb517ed58083af71bbedab8a4afbd1b0e84 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 11:45:22 +0200 Subject: [PATCH 4/9] :%s/self.out_path/self.base_path/g Signed-off-by: Nico Schottelius --- cdist/exec/local.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 6e4029f3..3a3ac706 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -43,16 +43,16 @@ class Local(object): target_host, exec_path=sys.argv[0], initial_manifest=None, - out_path=None, + base_path=None, add_conf_dirs=None): self.target_host = target_host # FIXME: stopped: create base that does not require moving later - if out_path: - self.out_path = out_path + if base_path: + self.base_path = base_path else: - self.out_path = tempfile.mkdtemp() + self.base_path = tempfile.mkdtemp() # FIXME: as well self._init_cache_dir(None) @@ -88,10 +88,10 @@ class Local(object): def _init_paths(self): # Depending on out_path - self.bin_path = os.path.join(self.out_path, "bin") - self.conf_path = os.path.join(self.out_path, "conf") - self.global_explorer_out_path = os.path.join(self.out_path, "explorer") - self.object_path = os.path.join(self.out_path, "object") + self.bin_path = os.path.join(self.base_path, "bin") + self.conf_path = os.path.join(self.base_path, "conf") + self.global_explorer_out_path = os.path.join(self.base_path, "explorer") + self.object_path = os.path.join(self.base_path, "object") # Depending on conf_path self.global_explorer_path = os.path.join(self.conf_path, "explorer") @@ -185,10 +185,10 @@ class Local(object): def save_cache(self): destination = os.path.join(self.cache_path, self.target_host) - self.log.debug("Saving " + self.out_path + " to " + destination) + self.log.debug("Saving " + self.base_path + " to " + destination) if os.path.exists(destination): shutil.rmtree(destination) - shutil.move(self.out_path, destination) + shutil.move(self.base_path, destination) def _create_conf_path_and_link_conf_dirs(self): # Link destination directories From ffeaa3d06bd8736e51a0074bb3c73c5998b8957d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 11:45:43 +0200 Subject: [PATCH 5/9] fix old bug / joining wrong args Signed-off-by: Nico Schottelius --- cdist/exec/remote.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 01f028a9..dfeccd94 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -149,6 +149,6 @@ class Remote(object): except subprocess.CalledProcessError: raise cdist.Error("Command failed: " + " ".join(command)) except OSError as error: - raise cdist.Error(" ".join(*args) + ": " + error.args[1]) + raise cdist.Error(" ".join(command) + ": " + error.args[1]) except UnicodeDecodeError: raise DecodeError(command) From 2f5de23ae9df42dd059f840ebd9a39fcf49aad96 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 12:03:25 +0200 Subject: [PATCH 6/9] out_path -> base_path Signed-off-by: Nico Schottelius --- cdist/core/manifest.py | 2 +- cdist/emulator.py | 15 ++++++++++----- cdist/test/code/__init__.py | 10 +++++----- cdist/test/emulator/__init__.py | 20 ++++++++++---------- cdist/test/explorer/__init__.py | 19 +++++++++---------- cdist/test/manifest/__init__.py | 8 ++++---- 6 files changed, 39 insertions(+), 35 deletions(-) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 8da7f96d..97121474 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -94,7 +94,7 @@ class Manifest(object): self.env = { 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), '__cdist_type_base_path': self.local.type_path, # for use in type emulator - '__global': self.local.out_path, + '__global': self.local.base_path, '__target_host': self.target_host, } if self.log.getEffectiveLevel() == logging.DEBUG: diff --git a/cdist/emulator.py b/cdist/emulator.py index add20e70..af4c620e 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -28,6 +28,8 @@ import sys import cdist from cdist import core +class MissingEnvironmentVariable(cdist.Error): + class Emulator(object): def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): self.argv = argv @@ -36,12 +38,15 @@ class Emulator(object): self.object_id = '' - self.global_path = self.env['__global'] - self.target_host = self.env['__target_host'] + try: + self.global_path = self.env['__global'] + self.target_host = self.env['__target_host'] - # Internally only - self.object_source = self.env['__cdist_manifest'] - self.type_base_path = self.env['__cdist_type_base_path'] + # Internally only + self.object_source = self.env['__cdist_manifest'] + self.type_base_path = self.env['__cdist_type_base_path'] + + except KeyError: self.object_base_path = os.path.join(self.global_path, "object") diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 284ef9c0..95a9e10f 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -41,11 +41,11 @@ class CodeTestCase(test.CdistTestCase): def setUp(self): self.target_host = 'localhost' - self.out_path = self.mkdtemp() + self.base_path = self.mkdtemp() self.local = local.Local( target_host=self.target_host, - out_path = self.out_path, + base_path = self.base_path, exec_path = cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -63,7 +63,7 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.create() def tearDown(self): - shutil.rmtree(self.out_path) + shutil.rmtree(self.base_path) shutil.rmtree(self.remote_base_path) def test_run_gencode_local_environment(self): @@ -75,7 +75,7 @@ class CodeTestCase(test.CdistTestCase): key = junk.split(' ')[1] output_dict[key] = value self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) @@ -90,7 +90,7 @@ class CodeTestCase(test.CdistTestCase): key = junk.split(' ')[1] output_dict[key] = value self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index ec0d7ae4..9f6d7000 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -33,7 +33,6 @@ from cdist.exec import local from cdist import emulator from cdist import core from cdist import config -import cdist.context import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -46,11 +45,11 @@ class EmulatorTestCase(test.CdistTestCase): self.temp_dir = self.mkdtemp() handle, self.script = self.mkstemp(dir=self.temp_dir) os.close(handle) - out_path = self.temp_dir + base_path = self.temp_dir self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -108,11 +107,11 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): def setUp(self): self.temp_dir = self.mkdtemp() - out_path = os.path.join(self.temp_dir, "out") + base_path = os.path.join(self.temp_dir, "out") self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -135,13 +134,13 @@ class ArgumentsTestCase(test.CdistTestCase): def setUp(self): self.temp_dir = self.mkdtemp() - out_path = self.temp_dir + base_path = self.temp_dir handle, self.script = self.mkstemp(dir=self.temp_dir) os.close(handle) self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -183,6 +182,7 @@ class ArgumentsTestCase(test.CdistTestCase): object_id = 'some-id' value = 'some value' argv = [type_name, object_id, '--required1', value, '--required2', value] + print(self.env) os.environ.update(self.env) emu = emulator.Emulator(argv) emu.run() @@ -227,11 +227,11 @@ class StdinTestCase(test.CdistTestCase): os.environ = os.environ.copy() self.temp_dir = self.mkdtemp() - out_path = os.path.join(self.temp_dir, "out") + base_path = os.path.join(self.temp_dir, "out") self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=base_path, exec_path=test.cdist_exec_path, add_conf_dirs=[conf_dir]) diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py index a97b538a..d1b86725 100644 --- a/cdist/test/explorer/__init__.py +++ b/cdist/test/explorer/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -39,26 +39,25 @@ conf_dir = op.join(fixtures, "conf") class ExplorerClassTestCase(test.CdistTestCase): def setUp(self): - self.target_host = 'localhost' - self.temp_dir = self.mkdtemp() - self.out_path = os.path.join(self.temp_dir, "out") + self.local_path = os.path.join(self.temp_dir, "local") self.remote_base_path = os.path.join(self.temp_dir, "remote") os.makedirs(self.remote_base_path) self.local = local.Local( target_host=self.target_host, - out_path=self.out_path, + base_path=self.local_path, exec_path=test.cdist_exec_path, - add_conf_dirs=[conf_dir]) + add_conf_dirs=[conf_dir], + ) self.local.create_files_dirs() self.remote = remote.Remote( - self.target_host, - self.remote_base_path, - self.remote_exec, - self.remote_copy) + target_host=self.target_host, + remote_exec=self.remote_exec, + remote_copy=self.remote_copy, + base_path=self.remote_base_path) self.remote.create_files_dirs() self.explorer = explorer.Explorer( diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index 727017e7..c375a80f 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -46,11 +46,11 @@ class ManifestTestCase(test.CdistTestCase): self.orig_environ = os.environ os.environ = os.environ.copy() self.temp_dir = self.mkdtemp() - self.target_host = 'localhost' + out_path = self.temp_dir self.local = local.Local( target_host=self.target_host, - out_path=out_path, + base_path=out_path, exec_path = cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -78,7 +78,7 @@ class ManifestTestCase(test.CdistTestCase): output_dict[key] = value self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) self.assertEqual(output_dict['__manifest'], self.local.manifest_path) @@ -99,7 +99,7 @@ class ManifestTestCase(test.CdistTestCase): output_dict[key] = value self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertEqual(output_dict['__target_host'], self.local.target_host) - self.assertEqual(output_dict['__global'], self.local.out_path) + self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) self.assertEqual(output_dict['__type'], cdist_type.absolute_path) self.assertEqual(output_dict['__object'], cdist_object.absolute_path) From 866645679a45cf2ec3fbf3b691e23782e48aaa0f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 13:34:29 +0200 Subject: [PATCH 7/9] throw a better exception when environment variables are missing Signed-off-by: Nico Schottelius --- cdist/emulator.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/cdist/emulator.py b/cdist/emulator.py index af4c620e..22c819cd 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -28,7 +28,14 @@ import sys import cdist from cdist import core -class MissingEnvironmentVariable(cdist.Error): +class MissingRequiredEnvironmentVariableError(cdist.Error): + def __init__(self, name): + self.name = name + self.message = "Emulator requires the environment variable %s to be setup" % self.name + + def __str__(self): + return self.message + class Emulator(object): def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): @@ -46,7 +53,8 @@ class Emulator(object): self.object_source = self.env['__cdist_manifest'] self.type_base_path = self.env['__cdist_type_base_path'] - except KeyError: + except KeyError as e: + raise MissingRequiredEnvironmentVariableError(e.args[0]) self.object_base_path = os.path.join(self.global_path, "object") From b527479620d38cf329bab54f16f84af738707e39 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 13:35:03 +0200 Subject: [PATCH 8/9] refactor out_path -> base_path Signed-off-by: Nico Schottelius --- cdist/core/code.py | 2 +- cdist/test/config_install/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index fa1ed3c1..d5f59094 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -89,7 +89,7 @@ class Code(object): self.remote = remote self.env = { '__target_host': self.target_host, - '__global': self.local.out_path, + '__global': self.local.base_path, } def _run_gencode(self, cdist_object, which): diff --git a/cdist/test/config_install/__init__.py b/cdist/test/config_install/__init__.py index 4047abe4..a66eaa13 100644 --- a/cdist/test/config_install/__init__.py +++ b/cdist/test/config_install/__init__.py @@ -47,11 +47,11 @@ class ConfigInstallRunTestCase(test.CdistTestCase): os.environ = os.environ.copy() self.temp_dir = self.mkdtemp() - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) + self.local_dir = os.path.join(self.temp_dir, "local") + os.mkdir(self.local_dir) self.local = cdist.exec.local.Local( target_host=self.target_host, - out_path=self.out_dir) + base_path=self.local_dir) self.remote_dir = os.path.join(self.temp_dir, "remote") os.mkdir(self.remote_dir) From 975b93c20a1530aec7f9be89fe1f793fe843056f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Aug 2013 13:37:40 +0200 Subject: [PATCH 9/9] fix all tests -> back to normal Signed-off-by: Nico Schottelius --- cdist/test/code/__init__.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 95a9e10f..fd0af338 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -39,21 +39,22 @@ conf_dir = op.join(fixtures, 'conf') class CodeTestCase(test.CdistTestCase): def setUp(self): - self.target_host = 'localhost' - - self.base_path = self.mkdtemp() + self.local_dir = self.mkdtemp() self.local = local.Local( target_host=self.target_host, - base_path = self.base_path, + base_path = self.local_dir, exec_path = cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() - self.remote_base_path = self.mkdtemp() + self.remote_dir = self.mkdtemp() remote_exec = self.remote_exec remote_copy = self.remote_copy - self.remote = remote.Remote(self.target_host, self.remote_base_path, remote_exec, remote_copy) + self.remote = remote.Remote(target_host=self.target_host, + base_path=self.remote_dir, + remote_exec=remote_exec, + remote_copy=remote_copy) self.remote.create_files_dirs() self.code = code.Code(self.target_host, self.local, self.remote) @@ -63,8 +64,8 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.create() def tearDown(self): - shutil.rmtree(self.base_path) - shutil.rmtree(self.remote_base_path) + shutil.rmtree(self.local_dir) + shutil.rmtree(self.remote_dir) def test_run_gencode_local_environment(self): output_string = self.code.run_gencode_local(self.cdist_object)