Merge remote-tracking branch 'telmich/master'

doc/dev/logs/2011-06-25.trigger-graphic

@@ -0,0 +1,15 @@
+
+
+
+
+
+
+client -------------------> trigger via ssh ----------> server
+                                                      ~/.ssh/authorized_keys
+                                                      -> cdist-deploy-to <da-wo-ich-herkomme>
+
+ssh -R 
+
+server -> via cdist-deploy-to -> client
+
+cdist-mass-deploy -p <host a .. host b...>

doc/dev/todo/tests

@@ -1,29 +1,139 @@
-Tests needed:
+Tests needed for config_install:
 
-   - Fail if cache_dir from previous run cannot be deleted
+   __init__():
-   - Fail if cache_dir cannot be created from current out_dir
+      - sets up env: __target_host
-   - transfer_type_explorers: Explorers are not transferred twice
-   - transfer_type_explorers: No transfer tried if there are no type_explorers
 
-   - does $require work?
+   cleanup()
-   - $whatever should fail if there is no global explorer directory
+      - Fail if cache_dir from previous run cannot be deleted
-   - emulator may only be called with __ as prefix - fail otherwise!
+      - Fail if cache_dir cannot be created from current out_dir
 
-   - ensure paths returned by object include dot-cdist
+   filter()
-   - ensure path of explorer of object returns correct path
+      - ensure logformat is changed: target host is prefixed:
+         LOGLEVEL: target_host: MESSAGE
 
-   config_install:
+   run_initial_manifest():
-      run_type_manifest() - same tests as for test_initial_manifest_*?
+      - parameter is actually used (from __init__)
-      run_manifest() - raise exception if manifest is not existent
+      - ensure changing the manifest actually runs a different manifest
-      object_run(): ensure no object is run twice
+      -> give ConfigInstall Constructor different manifest
-      object_run(): ensure requirements are taken into account
+         -> different manifest is executed.
-      object_run(): check (from extern?) that all needed variables are setup
+      - test all submitted (from core to type manifest) variables:
-      object_run(): ensure no code-{local, remote} is created, if gencode is not producing code
+         - ENVIRONMENT
-      object_run(): ensure no code-{local, remote} contains what gencode created
+         - they are set
+         - they contain the correct values
 
-      run_type_explorer(): ensure output is saved to correct path
+   run_type_manifest():
-      run_type_explorer(): ensure a type with no explorers works
+      - test all submitted (from core to type manifest) variables:
-      run_type_explorer(): ensure environment is setup correctly
+         - ENVIRONMENT
+         - they are set
+         - they contain the correct values
+      - same tests as for test_initial_manifest_*?
+
+   run_manifest():
+      - test all submitted variables:
+         - ENVIRONMENT
+            - including __debug, if debug
+         - they are set
+         - they contain the correct values
+      - does $require work?
+      - check that exception raised, if manifest is not existent
+
+   object_run():
+      - ensure no object is run twice
+      - ensure requirements are taken into account?
+         - and order of run is adjusted
+      - check (from extern?) that all needed variables are setup
+      - ensure no code-{local, remote} is created, 
+         if gencode is not producing code
+      - ensure THAT code-{local, remote} contains what gencode created
+      - abort if gencode-* fails
+      - abort if code-* fails
+         - abort == raise(FooException)
+      - gencode-*: ensure ENVIRONMENT is setup correctly
+
+   run_type_explorer()
+      - ensure ALL type explores have been run
+      - ensure output is saved to correct path
+      - ensure a type with {0,1,2} explorers works ?
+         - none, one, multiple
+      - ensure ENVIRONMENT is setup correctly
+      - fails if ANY of the given explorer fails
+
+   run_global_explorers():
+      - ensure ALL type explores have been run
+      - ensure output is saved to correct path
+      - ensure a type with {0,1,2} explorers works ?
+         - none, one, multiple
+      - ensure ENVIRONMENT is setup correctly
+      - fails if ANY of the given explorer fails
+      
+   link_emulator():
+      - ensure that links to ALL types are created
+      - ensure that links points to correct executable
+         - i.e. readlink() works
+         - AND target of readlink is the correct executable
+
+   stage_run()
+      - ensure that for every object in
+         cdist.core.Object.list_objects() is passed into
+         self.object_run()
+         - instantiate + overwrite + test
+
+   object_prepare():
+      - calls (in this order):
+         self.run_type_explorer(cdist_object)
+         self.run_type_manifest(cdist_object)
+         cdist_object.prepared = True
+      - object is prepared after function call
 
 
-      all: check that messages of all functions contain target_host in string
+   stage_prepare():
+      - calls (in this order):
+         self.link_emulator()
+         self.run_global_explorers()
+         self.run_initial_manifest()
+
+      - ensure that all objects are created :-)
+         - as defined in inital manifest + type manifests
+         
+   deploy_to()
+      - ensure self.stage_prepare() and self.stage_run() are
+         run. [ORDER]
+         - instantiate + overwrite + test
+
+   deploy_and_cleanup()
+      - ensure self.deploy_to() + self.cleanup() are are run
+         run. [ORDER]
+         - instantiate + overwrite + test
+
+   transfer_object_parameter()
+      - check that object parameters are transferred
+      - paths are setup correctly
+         - test via mock of remote_mkdir and transfer_path
+      - ensure self.remote_mkdir, self.transfer_path are run. [ORDER]
+         - instantiate + overwrite + test
+   
+   transfer_global_explorers()
+      - paths are setup correctly
+         - test via mock of self.remote_mkdir() and self.transfer_path()
+      - ensure self.remote_mkdir, self.transfer_path are run. [ORDER]
+         - instantiate + overwrite + test
+   
+   transfer_type_explorers():
+      - Explorers are not transferred twice
+      - paths are setup correctly
+         - test via mock of self.remote_mkdir() and self.transfer_path()
+      - Explorers are transferred
+         - test via mock of self.remote_mkdir() and self.transfer_path()
+
+   remote_mkdir()
+      - is directory created
+
+   remove_remote_path
+      - is path removed
+
+   transfer_path
+      - is src to dst transferred?
+
+emulator:
+   may only be called with __ as prefix - fail otherwise!
+

lib/cdist/config_install.py

@@ -49,20 +49,16 @@ class ConfigInstall:
 
         self.debug          = debug
 
-        # Required for testing
+        # Only required for testing
         self.exec_path      = exec_path
 
         # Configure logging
         log.addFilter(self)
 
         # Base and Temp Base 
-        if base_path:
+        self.base_path = (base_path or
-            self.base_path = base_path
+            self.base_path = os.path.abspath(os.path.join(
-        else:
+                os.path.dirname(__file__), os.pardir, os.pardir))
-            self.base_path = os.path.abspath(
-                os.path.join(os.path.dirname(__file__),
-                    os.pardir,
-                    os.pardir))
 
         # Local input
         self.cache_path             = os.path.join(self.base_path, "cache", 
@@ -74,10 +70,8 @@ class ConfigInstall:
         self.type_base_path         = os.path.join(self.conf_path, "type")
         self.lib_path               = os.path.join(self.base_path, "lib")
 
-        if initial_manifest:
+        self.initial_manifest = (initial_manifest or
-            self.initial_manifest = initial_manifest
+            os.path.join(self.manifest_path, "init"))
-        else:
-            self.initial_manifest = os.path.join(self.manifest_path, "init")
 
         # Local output
         if '__cdist_out_dir' in os.environ:
@@ -108,7 +102,6 @@ class ConfigInstall:
         self.__init_remote_paths()
 
 
-
     def __init_remote_paths(self):
         """Initialise remote directory structure"""
         self.remove_remote_path(self.remote_base_path)
@@ -122,6 +115,7 @@ class ConfigInstall:
         if not os.path.isdir(self.base_path):
             os.mkdir(self.base_path)
             
+        # FIXME: raise more beautiful exception / Steven: handle exception
         os.mkdir(self.out_path)
         os.mkdir(self.global_explorer_out_path)
         os.mkdir(self.bin_path)
@@ -129,6 +123,9 @@ class ConfigInstall:
     def __init_env(self):
         """Environment usable for other stuff"""
         os.environ['__target_host'] = self.target_host
+        if self.debug:
+            os.environ['__debug'] = "yes"
+
 
     def cleanup(self):
         # Do not use in __del__:
@@ -137,6 +134,7 @@ class ConfigInstall:
         # or in the process of being torn down (e.g. the import machinery shutting down)"
         #
         log.debug("Saving " + self.out_path + " to " + self.cache_path)
+        # FIXME: raise more beautiful exception / Steven: handle exception
         # Remove previous cache
         if os.path.exists(self.cache_path):
             shutil.rmtree(self.cache_path)
@@ -182,10 +180,6 @@ class ConfigInstall:
         env['__target_host']            = self.target_host
         env['__global']                 = self.out_path
         
-        # Submit debug flag to manifest, can be used by emulator and types
-        if self.debug:
-            env['__debug']              = "yes"
-
         # Required for recording source in emulator
         env['__cdist_manifest']         = manifest_path
 
@@ -198,6 +192,13 @@ class ConfigInstall:
 
         cdist.exec.shell_run_or_debug_fail(manifest_path, [manifest_path], env=env)
 
+    def object_prepare(self, cdist_object):
+        """Prepare object: Run type explorer + manifest"""
+        log.debug("Preparing object: " + cdist_object.name)
+        self.run_type_explorer(cdist_object)
+        self.run_type_manifest(cdist_object)
+        cdist_object.prepared = True
+
     def object_run(self, cdist_object):
         """Run gencode and code for an object"""
         log.debug("Running object %s", cdist_object)
@@ -235,6 +236,7 @@ class ConfigInstall:
                 outfile_fd = open(outfile, "w")
 
                 # Need to flush to ensure our write is done before stdout write
+                # FIXME: code header still needed?
                 outfile_fd.write(CODE_HEADER)
                 outfile_fd.flush()
 
@@ -299,7 +301,6 @@ class ConfigInstall:
             cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=True)
             output_fd.close()
 
-
     def link_emulator(self):
         """Link emulator to types"""
         src = os.path.abspath(self.exec_path)
@@ -307,7 +308,7 @@ class ConfigInstall:
             dst = os.path.join(self.bin_path, cdist_type.name)
             log.debug("Linking emulator: %s to %s", src, dst)
 
-            # FIXME: handle exception / make it more beautiful
+            # FIXME: handle exception / make it more beautiful / Steven: raise except :-)
             os.symlink(src, dst)
 
     def run_global_explorers(self):
@@ -329,15 +330,6 @@ class ConfigInstall:
             cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True)
             output_fd.close()
 
-
-    def stage_run(self):
-        """The final (and real) step of deployment"""
-        log.info("Generating and executing code")
-        for cdist_object in cdist.core.Object.list_objects(self.object_base_path,
-                                                           self.type_base_path):
-            log.debug("Run object: %s", cdist_object)
-            self.object_run(cdist_object)
-
     def deploy_to(self):
         """Mimic the old deploy to: Deploy to one host"""
         log.info("Deploying to " + self.target_host)
@@ -370,12 +362,17 @@ class ConfigInstall:
                     log.debug("Skipping rerun of object %s", cdist_object)
                     continue
                 else:
-                    log.debug("Preparing object: " + cdist_object.name)
+                    self.object_prepare(cdist_object)
-                    self.run_type_explorer(cdist_object)
-                    self.run_type_manifest(cdist_object)
-                    cdist_object.prepared = True
                     new_objects_created = True
 
+    def stage_run(self):
+        """The final (and real) step of deployment"""
+        log.info("Generating and executing code")
+        for cdist_object in cdist.core.Object.list_objects(self.object_base_path,
+                                                           self.type_base_path):
+            log.debug("Run object: %s", cdist_object)
+            self.object_run(cdist_object)
+
     def transfer_object_parameter(self, cdist_object):
         """Transfer the object parameter to the remote destination"""
         src  = os.path.join(self.object_base_path,