Merge branch 'enhance_cache'

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

Conflicts:
	cdist/test/code/__init__.py
	cdist/test/config_install/__init__.py
	cdist/test/explorer/__init__.py

cdist/config_install.py

@@ -117,7 +117,6 @@ class ConfigInstall(object):
         try:
             local = cdist.exec.local.Local(
                 target_host=host,
-                exec_path=sys.argv[0],
                 initial_manifest=args.manifest,
                 out_path=args.out_path,
                 add_conf_dirs=args.conf_dir)

cdist/core/code.py

@@ -89,7 +89,7 @@ class Code(object):
         self.remote = remote
         self.env = {
             '__target_host': self.target_host,
-            '__global': self.local.out_path,
+            '__global': self.local.base_path,
         }
 
     def _run_gencode(self, cdist_object, which):

cdist/core/manifest.py

@@ -94,7 +94,7 @@ class Manifest(object):
         self.env = {
             'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
             '__cdist_type_base_path': self.local.type_path, # for use in type emulator
-            '__global': self.local.out_path,
+            '__global': self.local.base_path,
             '__target_host': self.target_host,
         }
         if self.log.getEffectiveLevel() == logging.DEBUG:

cdist/emulator.py

@@ -28,6 +28,15 @@ import sys
 import cdist
 from cdist import core
 
+class MissingRequiredEnvironmentVariableError(cdist.Error):
+    def __init__(self, name):
+        self.name = name
+        self.message = "Emulator requires the environment variable %s to be setup" % self.name
+
+    def __str__(self):
+        return self.message
+
+
 class Emulator(object):
     def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
         self.argv           = argv
@@ -36,6 +45,7 @@ class Emulator(object):
 
         self.object_id      = ''
 
+        try:
             self.global_path    = self.env['__global']
             self.target_host    = self.env['__target_host']
 
@@ -43,6 +53,9 @@ class Emulator(object):
             self.object_source  = self.env['__cdist_manifest']
             self.type_base_path = self.env['__cdist_type_base_path']
 
+        except KeyError as e:
+            raise MissingRequiredEnvironmentVariableError(e.args[0])
+
         self.object_base_path = os.path.join(self.global_path, "object")
 
         self.type_name      = os.path.basename(argv[0])

cdist/exec/local.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2011 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -41,18 +41,18 @@ class Local(object):
     """
     def __init__(self,
                  target_host,
-                 exec_path,
+                 exec_path=sys.argv[0],
                  initial_manifest=None,
-                 out_path=None,
+                 base_path=None,
                  add_conf_dirs=None):
 
         self.target_host = target_host
 
         # FIXME: stopped: create base that does not require moving later
-        if out_path:
+        if base_path:
-            self.out_path = out_path
+            self.base_path = base_path
         else:
-            self.out_path = tempfile.mkdtemp()
+            self.base_path = tempfile.mkdtemp()
 
         # FIXME: as well
         self._init_cache_dir(None)
@@ -88,10 +88,10 @@ class Local(object):
 
     def _init_paths(self):
         # Depending on out_path
-        self.bin_path = os.path.join(self.out_path, "bin")
+        self.bin_path = os.path.join(self.base_path, "bin")
-        self.conf_path = os.path.join(self.out_path, "conf")
+        self.conf_path = os.path.join(self.base_path, "conf")
-        self.global_explorer_out_path = os.path.join(self.out_path, "explorer")
+        self.global_explorer_out_path = os.path.join(self.base_path, "explorer")
-        self.object_path = os.path.join(self.out_path, "object")
+        self.object_path = os.path.join(self.base_path, "object")
 
         # Depending on conf_path
         self.global_explorer_path = os.path.join(self.conf_path, "explorer")
@@ -185,10 +185,10 @@ class Local(object):
 
     def save_cache(self):
         destination = os.path.join(self.cache_path, self.target_host)
-        self.log.debug("Saving " + self.out_path + " to " + destination)
+        self.log.debug("Saving " + self.base_path + " to " + destination)
         if os.path.exists(destination):
             shutil.rmtree(destination)
-        shutil.move(self.out_path, destination)
+        shutil.move(self.base_path, destination)
 
     def _create_conf_path_and_link_conf_dirs(self):
         # Link destination directories

cdist/exec/remote.py

@@ -43,12 +43,20 @@ class Remote(object):
     Directly accessing the remote side from python code is a bug.
 
     """
-    def __init__(self, target_host, remote_exec, remote_copy):
+    def __init__(self,
+                 target_host,
+                 remote_exec,
+                 remote_copy,
+                 base_path=None):
         self.target_host = target_host
-        self.base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist")
         self._exec = remote_exec
         self._copy = remote_copy
 
+        if base_path:
+            self.base_path = base_path
+        else:
+            self.base_path = "/var/lib/cdist"
+
         self.conf_path = os.path.join(self.base_path, "conf")
         self.object_path = os.path.join(self.base_path, "object")
 
@@ -141,6 +149,6 @@ class Remote(object):
         except subprocess.CalledProcessError:
             raise cdist.Error("Command failed: " + " ".join(command))
         except OSError as error:
-            raise cdist.Error(" ".join(*args) + ": " + error.args[1])
+            raise cdist.Error(" ".join(command) + ": " + error.args[1])
         except UnicodeDecodeError:
             raise DecodeError(command)

cdist/test/code/__init__.py

@@ -39,18 +39,16 @@ conf_dir = op.join(fixtures, 'conf')
 class CodeTestCase(test.CdistTestCase):
 
     def setUp(self):
-        self.target_host = 'localhost'
+        self.local_dir = self.mkdtemp()
-
-        self.out_path = self.mkdtemp()
 
         self.local = local.Local(
             target_host=self.target_host, 
-            out_path = self.out_path,
+            base_path = self.local_dir,
             exec_path = cdist.test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
 
-        self.remote_base_path = self.mkdtemp()
+        self.remote_dir = self.mkdtemp()
         remote_exec = self.remote_exec
         remote_copy = self.remote_copy
         self.remote = remote.Remote(
@@ -58,7 +56,7 @@ class CodeTestCase(test.CdistTestCase):
             remote_exec=remote_exec, 
             remote_copy=remote_copy)
         self.remote.base_path = self.remote_base_path
-        self.remote.create_files_dirs()
+        tself.remote.create_files_dirs()
 
         self.code = code.Code(self.target_host, self.local, self.remote)
 
@@ -67,8 +65,8 @@ class CodeTestCase(test.CdistTestCase):
         self.cdist_object.create()
 
     def tearDown(self):
-        shutil.rmtree(self.out_path)
+        shutil.rmtree(self.local_dir)
-        shutil.rmtree(self.remote_base_path)
+        shutil.rmtree(self.remote_dir)
 
     def test_run_gencode_local_environment(self):
         output_string = self.code.run_gencode_local(self.cdist_object)
@@ -79,7 +77,7 @@ class CodeTestCase(test.CdistTestCase):
                 key = junk.split(' ')[1]
                 output_dict[key] = value
         self.assertEqual(output_dict['__target_host'], self.local.target_host)
-        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__global'], self.local.base_path)
         self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
         self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
         self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
@@ -94,7 +92,7 @@ class CodeTestCase(test.CdistTestCase):
                 key = junk.split(' ')[1]
                 output_dict[key] = value
         self.assertEqual(output_dict['__target_host'], self.local.target_host)
-        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__global'], self.local.base_path)
         self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
         self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
         self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)

cdist/test/config_install/__init__.py

@@ -47,19 +47,19 @@ class ConfigInstallRunTestCase(test.CdistTestCase):
         os.environ = os.environ.copy()
         self.temp_dir = self.mkdtemp()
 
-        self.out_dir = os.path.join(self.temp_dir, "out")
+        self.local_dir = os.path.join(self.temp_dir, "local")
-        self.remote_out_dir = os.path.join(self.temp_dir, "remote")
+        os.mkdir(self.local_dir)
-
+        self.local = cdist.exec.local.Local(
-        os.environ['__cdist_out_dir'] = self.out_dir
-        os.environ['__cdist_remote_out_dir'] = self.remote_out_dir
-
-        self.local = cdist.exec.local(
             target_host=self.target_host,
-            exec_path=test.cdist_exec_path)
+            base_path=self.local_dir)
-        self.remote = cdist.exec.remote(
+
+        self.remote_dir = os.path.join(self.temp_dir, "remote")
+        os.mkdir(self.remote_dir)
+        self.remote = cdist.exec.remote.Remote(
             target_host=self.target_host,
             remote_copy=self.remote_copy,
-            remote_exec=self.remote_exec)
+            remote_exec=self.remote_exec,
+            base_path=self.remote_dir)
 
         self.local.object_path  = object_base_path
         self.local.type_path    = type_base_path

cdist/test/emulator/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -45,11 +45,11 @@ class EmulatorTestCase(test.CdistTestCase):
         self.temp_dir = self.mkdtemp()
         handle, self.script = self.mkstemp(dir=self.temp_dir)
         os.close(handle)
-        out_path = self.temp_dir
+        base_path = self.temp_dir
 
         self.local = local.Local(
             target_host=self.target_host,
-            out_path=out_path,
+            base_path=base_path,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -107,11 +107,11 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
 
     def setUp(self):
         self.temp_dir = self.mkdtemp()
-        out_path = os.path.join(self.temp_dir, "out")
+        base_path = os.path.join(self.temp_dir, "out")
 
         self.local = local.Local(
             target_host=self.target_host,
-            out_path=out_path,
+            base_path=base_path,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -134,13 +134,13 @@ class ArgumentsTestCase(test.CdistTestCase):
 
     def setUp(self):
         self.temp_dir = self.mkdtemp()
-        out_path = self.temp_dir
+        base_path = self.temp_dir
         handle, self.script = self.mkstemp(dir=self.temp_dir)
         os.close(handle)
 
         self.local = local.Local(
             target_host=self.target_host,
-            out_path=out_path,
+            base_path=base_path,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -182,6 +182,7 @@ class ArgumentsTestCase(test.CdistTestCase):
         object_id = 'some-id'
         value = 'some value'
         argv = [type_name, object_id, '--required1', value, '--required2', value]
+        print(self.env)
         os.environ.update(self.env)
         emu = emulator.Emulator(argv)
         emu.run()
@@ -226,11 +227,11 @@ class StdinTestCase(test.CdistTestCase):
         os.environ = os.environ.copy()
 
         self.temp_dir = self.mkdtemp()
-        out_path = os.path.join(self.temp_dir, "out")
+        base_path = os.path.join(self.temp_dir, "out")
 
         self.local = local.Local(
             target_host=self.target_host,
-            out_path=out_path,
+            base_path=base_path,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
 

cdist/test/explorer/__init__.py

@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 #
 # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
-# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -39,25 +39,25 @@ conf_dir = op.join(fixtures, "conf")
 class ExplorerClassTestCase(test.CdistTestCase):
 
     def setUp(self):
-        self.target_host = 'localhost'
-
         self.temp_dir = self.mkdtemp()
-        self.out_path = os.path.join(self.temp_dir, "out")
+        self.local_path = os.path.join(self.temp_dir, "local")
         self.remote_base_path = os.path.join(self.temp_dir, "remote")
         os.makedirs(self.remote_base_path)
 
         self.local = local.Local(
             target_host=self.target_host,
-            out_path=self.out_path,
+            base_path=self.local_path,
             exec_path=test.cdist_exec_path,
-            add_conf_dirs=[conf_dir])
+            add_conf_dirs=[conf_dir],
+            )
 
         self.local.create_files_dirs()
 
         self.remote = remote.Remote(
-            self.target_host, 
+            target_host=self.target_host, 
-            self.remote_exec,
+            remote_exec=self.remote_exec,
-            self.remote_copy)
+            remote_copy=self.remote_copy,
+            base_path=self.remote_base_path)
         self.remote.create_files_dirs()
 
         self.explorer = explorer.Explorer(

cdist/test/manifest/__init__.py

@@ -46,11 +46,11 @@ class ManifestTestCase(test.CdistTestCase):
         self.orig_environ = os.environ
         os.environ = os.environ.copy()
         self.temp_dir = self.mkdtemp()
-        self.target_host = 'localhost'
+
         out_path = self.temp_dir
         self.local = local.Local(
             target_host=self.target_host,
-            out_path=out_path,
+            base_path=out_path,
             exec_path = cdist.test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -78,7 +78,7 @@ class ManifestTestCase(test.CdistTestCase):
                 output_dict[key] = value
         self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
         self.assertEqual(output_dict['__target_host'], self.local.target_host)
-        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__global'], self.local.base_path)
         self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path)
         self.assertEqual(output_dict['__manifest'], self.local.manifest_path)
 
@@ -99,7 +99,7 @@ class ManifestTestCase(test.CdistTestCase):
                 output_dict[key] = value
         self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path))
         self.assertEqual(output_dict['__target_host'], self.local.target_host)
-        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__global'], self.local.base_path)
         self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path)
         self.assertEqual(output_dict['__type'], cdist_type.absolute_path)
         self.assertEqual(output_dict['__object'], cdist_object.absolute_path)

scripts/cdist

@@ -65,8 +65,9 @@ def commandline():
          dest='manifest', required=False)
     parser['configinstall'].add_argument('-n', '--dry-run',
          help='Do not execute code', action='store_true')
-    parser['configinstall'].add_argument('-o', '--out-path',
+    parser['configinstall'].add_argument('-o', '--out-dir',
-         help='Directory prefix to save cdist output in')
+         help='Directory to save cdist output in',
+         destination="out_path")
     parser['configinstall'].add_argument('-p', '--parallel',
          help='Operate on multiple hosts in parallel',
          action='store_true', dest='parallel')