Merge remote-tracking branch 'steven/master'

bin/cdist

@@ -114,8 +114,9 @@ def configinstall(args, mode):
 
     time_start = time.time()
 
+    import cdist.context
+
     for host in args.host:
-        # Setup Local/Remote
         context = cdist.context.Context(
             target_host=host,
             initial_manifest=args.manifest,

lib/cdist/config_install.py

@@ -28,11 +28,8 @@ import sys
 import tempfile
 import time
 
-import cdist.core
+from cdist import core
-import cdist.context
+
-import cdist.exec
-#import cdist.explorer
-#import cdist.manifest
 
 class ConfigInstall(object):
     """Cdist main class to hold arbitrary data"""
@@ -41,58 +38,26 @@ class ConfigInstall(object):
 
         self.context = context
         self.log = logging.getLogger(self.context.target_host)
-        self.exec_wrapper   = cdist.exec.Wrapper(
-            target_host = self.context.target_host,
-            remote_exec=self.context.remote_exec,
-            remote_copy=self.context.remote_copy)
 
-        # Create directories other may depend on
+        # For easy access
-        self.__init_local_paths()
+        self.local = context.local
-        self.__init_remote_paths()
+        self.remote = context.remote
 
-        self.global_explorer = cdist.explorer.GlobalExplorer(self.context.global_in, out)
+        # Initialise local directory structure
-        self.type_explorer = cdist.explorer.GlobalExplorer(self.context.global_in, out)
+        self.local.create_directories()
+        # Initialise remote directory structure
+        self.remote.create_directories()
 
-        self.global_explorer = cdist.core.GlobalExplorer
+        self.explorer = core.Explorer(self.context.target_host, self.local, self.remote)
-        #self.manifest = cdist.manifest.Mamifest()
+        self.manifest = core.Manifest(self.context.target_host, self.local)
-
+        self.code = core.Code(self.context.target_host, self.local, self.remote)
-        self.manifest.initial_manifest()
-        self.manifest.type_manifest(cdist_object)
-        self.global_explorer.run()?
-
-        self.type_explorer.run(cdist_object)?
-
-        self.log = logging.getLogger(self.context.target_host)
-
-        # Setup env to be used by others - FIXME
-        self.__init_env()
-
-    def __init_remote_paths(self):
-        """Initialise remote directory structure"""
-        self.exec_wrapper.remove_remote_path(self.context.remote_base_path)
-        self.exec_wrapper.remote_mkdir(self.context.remote_base_path)
-        self.exec_wrapper.remote_mkdir(self.context.remote_conf_path)
-
-    def __init_local_paths(self):
-        """Initialise local directory structure"""
-
-        # Create base dir, if user supplied and not existing
-        if not os.path.isdir(self.context.base_path):
-            os.mkdir(self.context.base_path)
-            
-        # FIXME: raise more beautiful exception / Steven: handle exception
-        os.mkdir(self.context.out_path)
-        os.mkdir(self.context.bin_path)
-
-#        os.environ['__target_host'] = self.context.target_host
-#        if self.context.debug:
-#            os.environ['__debug'] = "yes"
 
     def cleanup(self):
-        self.log.debug("Saving " + self.context.out_path + " to " + self.context.cache_path)
+        # FIXME: move to local?
-        if os.path.exists(self.context.cache_path):
+        self.log.debug("Saving " + self.local.out_path + " to " + self.local.cache_path)
-            shutil.rmtree(self.context.cache_path)
+        if os.path.exists(self.local.cache_path):
-        shutil.move(self.context.out_path, self.context.cache_path)
+            shutil.rmtree(self.local.cache_path)
+        shutil.move(self.local.out_path, self.local.cache_path)
 
     def deploy_to(self):
         """Mimic the old deploy to: Deploy to one host"""
@@ -110,9 +75,9 @@ class ConfigInstall(object):
 
     def stage_prepare(self):
         """Do everything for a deploy, minus the actual code stage"""
-        self.link_emulator()
+        self.local.link_emulator(self.context.exec_path)
-        self.global_explorer_run()
+        self.run_global_explorers()
-        self.run_initial_manifest()
+        self.manifest.run_initial_manifest(self.context.initial_manifest)
 
         self.log.info("Running object manifests and type explorers")
 
@@ -120,8 +85,8 @@ class ConfigInstall(object):
         new_objects_created = True
         while new_objects_created:
             new_objects_created = False
-            for cdist_object in cdist.core.Object.list_objects(self.object_base_path,
+            for cdist_object in core.Object.list_objects(self.local.object_path,
-                                                               self.context.type_base_path):
+                                                         self.local.type_path):
                 if cdist_object.prepared:
                     self.log.debug("Skipping rerun of object %s", cdist_object)
                     continue
@@ -129,22 +94,27 @@ class ConfigInstall(object):
                     self.object_prepare(cdist_object)
                     new_objects_created = True
 
-
+    def run_global_explorers(self):
-    def self.global_explorer_run()
         """Run global explorers and save output"""
+        # FIXME: move to explorer, pass global_explorer_out_path as argument
+        self.explorer.transfer_global_explorers()
+        for explorer in self.explorer.list_global_explorer_names():
+            output = self.explorer.run_global_explorer(explorer)
+            path = os.path.join(self.local.global_explorer_out_path, explorer)
+            with open(path, 'w') as fd:
+                fd.write(output)
 
-        output = self.global_explorer.run()
+    def run_type_explorers(self, cdist_object):
-
+        """Run type explorers and save output in object."""
-        for explorer in output:
+        self.explorer.transfer_type_explorers(cdist_object.type)
-            outfile = os.path.join(self.context.global_explorer_out_path,
+        for explorer in self.explorer.list_type_explorer_names(cdist_object.type):
-                explorer)
+            output = self.explorer.run_type_explorer(explorer, cdist_object)
-            outfile_fd = open(outfile, "w")
+            cdist_object.explorers[explorer] = output
-            outfile_fd.write()
 
     def object_prepare(self, cdist_object):
         """Prepare object: Run type explorer + manifest"""
         self.log.debug("Preparing object: " + cdist_object.name)
-        cdist_object.explorers = self.explorer.run_type_explorer(cdist_object)
+        self.run_type_explorers(cdist_object)
         self.manifest.run_type_manifest(cdist_object)
         cdist_object.prepared = True
 
@@ -160,78 +130,26 @@ class ConfigInstall(object):
             
         for requirement in cdist_object.requirements:
             self.log.debug("Object %s requires %s", cdist_object, requirement)
+            # FIXME: requirement is a string, need to create object here
             self.object_run(requirement)
 
-        #
+        # Generate
-        # Setup env Variable:
+        cdist_object.code_local = self.code.run_gencode_local(cdist_object)
-        #
+        cdist_object.code_remote = self.code.run_gencode_remote(cdist_object)
-        env = os.environ.copy()
+        if cdist_object.code_local or cdist_object.code_remote:
-        env['__target_host']    = self.context.target_host
+            cdist_object.changed = True
-        env['__global']         = self.context.out_path
-        env["__object"]         = os.path.join(self.object_base_path, cdist_object.path)
-        env["__object_id"]      = cdist_object.object_id
-        env["__object_fq"]      = cdist_object.name
-        env["__type"]           = cdist_type.name
 
-        # gencode
+        # Execute
-        for cmd in ["local", "remote"]:
+        if cdist_object.code_local:
-            bin = os.path.join(self.context.type_base_path,
+            self.code.run_code_local(cdist_object)
-                    getattr(cdist_type, "gencode_" + cmd + "_path"))
+        if cdist_object.code_remote:
-
+            self.code.transfer_code_remote(cdist_object)
-            if os.path.isfile(bin):
+            self.code.run_code_remote(cdist_object)
-                outfile = os.path.join(self.object_base_path,
-                            getattr(cdist_object, "code_" + cmd + "_path"))
-
-                outfile_fd = open(outfile, "w")
-
-                # Need to flush to ensure our write is done before stdout write
-                # FIXME: code header still needed?
-                outfile_fd.write(CODE_HEADER)
-                outfile_fd.flush()
-
-                cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd)
-                outfile_fd.close()
-
-                status = os.stat(outfile)
-
-                # Remove output if empty, else make it executable
-                if status.st_size == len(CODE_HEADER):
-                    os.unlink(outfile)
-                else:
-                    # Add header and make executable - identically to 0o700
-                    os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR)
-                    cdist_object.changed=True
-
-        # code local
-        code_local = cdist_object.code_local_path
-        if os.path.isfile(code_local):
-            cdist.exec.run_or_fail([code_local])
-
-        # code remote
-        local_remote_code   = os.path.join(self.object_base_path,
-            cdist_object.code_remote_path)
-        remote_remote_code  = os.path.join(self.remote_object_path,
-            cdist_object.code_remote_path)
-        if os.path.isfile(local_remote_code):
-            self.context.transfer_path(local_remote_code, remote_remote_code)
-            cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True)
-
-        cdist_object.ran = True
-
-    def link_emulator(self):
-        """Link emulator to types"""
-        src = os.path.abspath(self.context.exec_path)
-        for cdist_type in cdist.core.Type.list_types(self.context.type_base_path):
-            dst = os.path.join(self.context.bin_path, cdist_type.name)
-            self.log.debug("Linking emulator: %s to %s", src, dst)
-
-            # FIXME: handle exception / make it more beautiful / Steven: raise except :-)
-            os.symlink(src, dst)
 
     def stage_run(self):
         """The final (and real) step of deployment"""
         self.log.info("Generating and executing code")
-        for cdist_object in cdist.core.Object.list_objects(self.object_base_path,
+        for cdist_object in core.Object.list_objects(self.local.object_path,
-                                                           self.context.type_base_path):
+                                                           self.local.type_path):
             self.log.debug("Run object: %s", cdist_object)
             self.object_run(cdist_object)

lib/cdist/context.py

@@ -24,12 +24,11 @@ import logging
 import os
 import sys
 import tempfile
-#import stat
+import shutil
-#import shutil
+
-#import time
+from cdist.exec import local
-#
+from cdist.exec import remote
-#import cdist.core
+
-#import cdist.exec
 
 class Context(object):
     """Hold information about current context"""
@@ -52,25 +51,13 @@ class Context(object):
         self.log = logging.getLogger(self.target_host)
         self.log.addFilter(self)
 
-        # Base and Temp Base 
+        # Local base directory
         self.base_path = (base_path or
             os.path.abspath(os.path.join(os.path.dirname(__file__),
                 os.pardir, os.pardir)))
 
-        # Local input
+        # Local temp directory
-        self.cache_path             = os.path.join(self.base_path, "cache", 
+        # FIXME: if __cdist_out_dir can be given from the outside, the same directory will be used for all hosts
-            self.target_host)
-        self.conf_path              = os.path.join(self.base_path, "conf")
-
-        self.global_explorer_path   = os.path.join(self.conf_path, "explorer")
-        self.manifest_path          = os.path.join(self.conf_path, "manifest")
-        self.type_base_path         = os.path.join(self.conf_path, "type")
-        self.lib_path               = os.path.join(self.base_path, "lib")
-
-        self.initial_manifest = (initial_manifest or
-            os.path.join(self.manifest_path, "init"))
-
-        # Local output
         if '__cdist_out_dir' in os.environ:
             self.out_path = os.environ['__cdist_out_dir']
             self.temp_dir = None
@@ -78,31 +65,17 @@ class Context(object):
             self.temp_dir = tempfile.mkdtemp()
             self.out_path = os.path.join(self.temp_dir, "out")
 
-        self.bin_path                 = os.path.join(self.out_path, "bin")
+        self.local = local.Local(self.target_host, self.base_path, self.out_path)
-        self.global_explorer_out_path = os.path.join(self.out_path, "explorer")
-        self.object_base_path         = os.path.join(self.out_path, "object")
 
-        # Remote directory base
+        self.initial_manifest = (initial_manifest or
-        if '__cdist_remote_out_dir' in os.environ:
+            os.path.join(self.local.manifest_path, "init"))
-            self.remote_base_path = os.environ['__cdist_remote_out_dir']
-        else:
-            self.remote_base_path = "/var/lib/cdist"
 
-        self.remote_conf_path            = os.path.join(self.remote_base_path, "conf")
+        # Remote
-        self.remote_object_path          = os.path.join(self.remote_base_path, "object")
+        self.remote_base_path = os.environ.get('__cdist_remote_out_dir', "/var/lib/cdist")
-
+        self.remote_exec = os.environ.get('__remote_exec', "ssh -o User=root -q")
-        self.remote_type_path            = os.path.join(self.remote_conf_path, "type")
+        self.remote_copy = os.environ.get('__remote_copy', "scp -o User=root -q")
-        self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer")
+        self.remote = remote.Remote(self.target_host, self.remote_base_path, 
-
+            self.remote_exec, self.remote_copy)
-        if '__remote_exec' in os.environ:
-            self.remote_exec = os.environ['__remote_exec']
-        else:
-            self.remote_exec = "ssh -o User=root -q"
-
-        if '__remote_copy' in os.environ:
-            self.remote_copy = os.environ['__remote_copy']
-        else:
-            self.remote_copy = "scp -o User=root -q"
 
     def cleanup(self):
         """Remove temp stuff"""

lib/cdist/core/__init__.py

@@ -19,8 +19,8 @@
 #
 #
 
-__all__ = ['Type', 'Object']
-
 from cdist.core.type import Type
 from cdist.core.object import Object
-from cdist.core.global_explorer import GlobalExplorer
+from cdist.core.explorer import Explorer
+from cdist.core.manifest import Manifest
+from cdist.core.code import Code

lib/cdist/core/code.py

@@ -95,6 +95,7 @@ class Code(object):
     def _run_gencode(self, cdist_object, which):
         cdist_type = cdist_object.type
         script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which))
+        if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
             env.update({
@@ -122,7 +123,7 @@ class Code(object):
 
     def _run_code(self, cdist_object, which):
         which_exec = getattr(self, which)
-        script = os.path.join(self.local.object_path, getattr(cdist_object, 'code_%s_path' % which))
+        script = os.path.join(which_exec.object_path, getattr(cdist_object, 'code_%s_path' % which))
         return which_exec.run_script(script)
 
     def run_code_local(self, cdist_object):

lib/cdist/core/explorer.py

@@ -95,10 +95,14 @@ class Explorer(object):
     def list_type_explorer_names(self, cdist_type):
         """Return a list of explorer names for the given type."""
         source = os.path.join(self.local.type_path, cdist_type.explorer_path)
+        try:
             return os.listdir(source)
+        except EnvironmentError:
+            return []
 
     def transfer_type_explorers(self, cdist_type):
         """Transfer the type explorers for the given type to the remote side."""
+        if cdist_type.explorers:
             source = os.path.join(self.local.type_path, cdist_type.explorer_path)
             destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
             self.remote.mkdir(destination)

lib/cdist/core/manifest.py

@@ -79,9 +79,12 @@ class Manifest(object):
         env = os.environ.copy()
         env.update(self.env)
         env['__manifest'] = self.local.manifest_path
+        env['__cdist_manifest'] = script
         return self.local.run_script(script, env=env)
 
     def run_type_manifest(self, cdist_object):
+        script = os.path.join(self.local.type_path, cdist_object.type.manifest_path)
+        if os.path.isfile(script):
             env = os.environ.copy()
             env.update(self.env)
             env.update({
@@ -89,6 +92,6 @@ class Manifest(object):
                 '__object_id': cdist_object.object_id,
                 '__object_fq': cdist_object.path,
                 '__type': cdist_object.type.absolute_path,
+                '__cdist_manifest': script,
             })
-        script = os.path.join(self.local.type_path, cdist_object.type.manifest_path)
             return self.local.run_script(script, env=env)

lib/cdist/exec/local.py

@@ -73,6 +73,7 @@ class Local(object):
     
     def create_directories(self):
         self.mkdir(self.out_path)
+        self.mkdir(self.global_explorer_out_path)
         self.mkdir(self.bin_path)
 
     def rmdir(self, path):
@@ -83,6 +84,7 @@ class Local(object):
     def mkdir(self, path):
         """Create directory on the local side."""
         self.log.debug("Local mkdir: %s", path)
+        # FIXME: dont set mode here, fix unittest mkdtemp instead
         os.makedirs(path, mode=0o700, exist_ok=True)
 
     def run(self, command, env=None):

lib/cdist/test/code/__init__.py

@@ -37,7 +37,7 @@ my_dir = op.abspath(op.dirname(__file__))
 fixtures = op.join(my_dir, 'fixtures')
 local_base_path = fixtures
 
-class ExplorerClassTestCase(unittest.TestCase):
+class CodeTestCase(unittest.TestCase):
 
     def mkdtemp(self, **kwargs):
         return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)
@@ -134,33 +134,3 @@ class ExplorerClassTestCase(unittest.TestCase):
         self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
         self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
         self.assertEqual(output_dict['__object_fq'], self.cdist_object.path)
-
-'''
-    def test_list_type_explorer_names(self):
-        cdist_type = core.Type(self.local.type_path, '__test_type')
-        expected = cdist_type.explorers
-        self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected)
-
-    def test_transfer_type_explorers(self):
-        cdist_type = core.Type(self.local.type_path, '__test_type')
-        self.explorer.transfer_type_explorers(cdist_type)
-        source = os.path.join(self.local.type_path, cdist_type.explorer_path)
-        destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
-        self.assertEqual(os.listdir(source), os.listdir(destination))
-
-    def test_transfer_object_parameters(self):
-        cdist_type = core.Type(self.local.type_path, '__test_type')
-        cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
-        cdist_object.parameters = {'first': 'first value', 'second': 'second value'}
-        self.explorer.transfer_object_parameters(cdist_object)
-        source = os.path.join(self.local.object_path, cdist_object.parameter_path)
-        destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
-        self.assertEqual(os.listdir(source), os.listdir(destination))
-
-    def test_run_type_explorer(self):
-        cdist_type = core.Type(self.local.type_path, '__test_type')
-        cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
-        self.explorer.transfer_type_explorers(cdist_type)
-        output = self.explorer.run_type_explorer('world', cdist_object)
-        self.assertEqual(output, 'hello\n')
-'''