Merge remote-tracking branch 'telmich/master'

lib/cdist/__init__.py

@@ -22,6 +22,8 @@
 VERSION     = "2.0.3"
 DOT_CDIST   = ".cdist"
 
+import os
+
 class Error(Exception):
     """Base exception class for this project"""
     pass

lib/cdist/config.py

@@ -42,7 +42,7 @@ def config(args):
     os.environ['__remote_copy'] = "scp -o User=root -q"
 
     for host in args.host:
-        c = Config(host, initial_manifest=args.manifest, base_dir=args.cdist_home, debug=args.debug)
+        c = Config(host, initial_manifest=args.manifest, base_path=args.cdist_home, debug=args.debug)
         if args.parallel:
             log.debug("Creating child process for %s", host)
             process[host] = multiprocessing.Process(target=c.deploy_and_cleanup)

lib/cdist/config_install.py

@@ -48,15 +48,13 @@ class ConfigInstall:
         self.exec_path      = exec_path
 
         self.context = cdist.context.Context(self.target_host,
-            initial_manifest=initial_manifest, base_path=base_path,
+            initial_manifest=initial_manifest,
+            base_path=base_path,
             debug=debug)
-        
+
     def cleanup(self):
         self.path.cleanup()
 
-    def __init_env(self):
-        """Setup environment"""
-
     def run_initial_manifest(self):
         """Run the initial manifest"""
         log.info("Running initial manifest %s", self.context.initial_manifest)
@@ -72,7 +70,7 @@ class ConfigInstall:
         log.debug("%s: Running %s", cdist_object.name, manifest)
         if os.path.exists(manifest_path):
             env = { "__object" :    os.path.join(self.context.object_base_path,
-                                        cdist_object.path)
+                                        cdist_object.path),
                     "__object_id":  cdist_object.object_id,
                     "__object_fq":  cdist_object.name,
                     "__type":       os.path.join(self.context.type_base_path,
@@ -98,7 +96,7 @@ class ConfigInstall:
         env['__cdist_manifest']         = manifest_path
 
         # Required to find types in emulator
-        env['__cdist_type_base_path']    = type.path
+        env['__cdist_type_base_path']    = self.context.type_base_path
 
         # Other environment stuff
         if extra_env:
@@ -204,7 +202,7 @@ class ConfigInstall:
     def link_emulator(self):
         """Link emulator to types"""
         src = os.path.abspath(self.exec_path)
-        for type in cdist.core.Type.list_types():
+        for type in cdist.core.Type.list_types(self.context.type_base_path):
             dst = os.path.join(self.context.bin_path, type.name)
             log.debug("Linking emulator: %s to %s", src, dst)
 
@@ -215,16 +213,17 @@ class ConfigInstall:
         """Run global explorers"""
         log.info("Running global explorers")
 
-        src = cdist.core.GlobalExplorer.base_path
+        src_path = self.context.global_explorer_path
-        dst = cdist.core.GlobalExplorer.remote_base_path
+        dst_path = self.context.global_explorer_out_path
+        remote_dst_path = self.context.remote_global_explorer_path
 
-        self.context.transfer_path(src, dst)
+        self.context.transfer_path(src_path, remote_dst_path)
 
-        for explorer in cdist.core.GlobalExplorer.list_explorers():
+        for explorer in os.listdir(src_path):
-            output_fd = open(explorer.out_path, mode='w')
+            output_fd = open(os.path.join(dst_path, explorer), mode='w')
             cmd = []
-            cmd.append("__explorer=" + cdist.core.GlobalExplorer.remote_base_path)
+            cmd.append("__explorer=" + remote_dst_path)
-            cmd.append(explorer.remote_path)
+            cmd.append(os.path.join(src_path, explorer))
 
             cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True)
             output_fd.close()
@@ -233,7 +232,7 @@ class ConfigInstall:
     def stage_run(self):
         """The final (and real) step of deployment"""
         log.info("Generating and executing code")
-        for cdist_object in cdist.core.Object.list_objects():
+        for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path):
             log.debug("Run object: %s", cdist_object)
             self.object_run(cdist_object)
 
@@ -248,29 +247,19 @@ class ConfigInstall:
         self.deploy_to()
         self.cleanup()
 
-    def init_deploy(self):
-        """Ensure the base directories are cleaned up"""
-        log.debug("Creating clean directory structure")
-
-        self.context.remove_remote_path(self.context.remote_base_path)
-        self.context.remote_mkdir(self.context.remote_base_path)
-        self.link_emulator()
-    
     def stage_prepare(self):
         """Do everything for a deploy, minus the actual code stage"""
-        self.init_deploy()
+        self.link_emulator()
         self.run_global_explorers()
         self.run_initial_manifest()
         
         log.info("Running object manifests and type explorers")
 
-        log.debug("Searching for objects in " + cdist.core.Object.base_path())
-
         # Continue process until no new objects are created anymore
         new_objects_created = True
         while new_objects_created:
             new_objects_created = False
-            for cdist_object in cdist.core.Object.list_objects():
+            for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path):
                 if cdist_object.prepared:
                     log.debug("Skipping rerun of object %s", cdist_object)
                     continue
@@ -281,26 +270,16 @@ class ConfigInstall:
                     cdist_object.prepared = True
                     new_objects_created = True
 
-    # FIXME Move into configinstall
     def transfer_object_parameter(self, cdist_object):
         """Transfer the object parameter to the remote destination"""
-        local_path  = 
+        src  = os.path.join(self.context.object_base_path,
-            os.path.join(self.context.object_base_path,
+            cdist_object.parameter_path)
-                cdist_object.parameter_path)
+        dst = os.path.join(self.context.remote_object_path,
-        remote_path = 
+            cdist_object.parameter_path)
-            os.path.join(self.context.remote_object_path,
-                cdist_object.parameter_path)
-
-        # Create base path before using mkdir -p
-        # FIXME: needed?
-        self.remote_mkdir(remote_path)
 
         # Synchronise parameter dir afterwards
         self.transfer_path(local_path, remote_path)
 
-
-####FIXED ######################################################################
-
     def transfer_global_explorers(self):
         """Transfer the global explorers"""
         self.remote_mkdir(self.context.remote_global_explorer_path)

lib/cdist/context.py

@@ -47,11 +47,10 @@ class Context:
         if base_path:
             self.base_path = base_path
         else:
-            self.base_path = 
+            self.base_path = os.path.abspath(
-                os.path.abspath(
+                os.path.join(os.path.dirname(__file__),
-                    os.path.join(os.path.dirname(__file__),
+                    os.pardir,
-                        os.pardir,
+                    os.pardir))
-                        os.pardir))
         
 
         # Local input directories
@@ -70,7 +69,7 @@ class Context:
 
         # Local output directories
         if out_path:
-            self.out_path = out_path:
+            self.out_path = out_path
         else:
             self.out_path = os.path.join(tempfile.mkdtemp(), "out")
 
@@ -86,13 +85,13 @@ class Context:
 
         self.remote_conf_path            = os.path.join(self.remote_base_path, "conf")
         self.remote_object_path          = os.path.join(self.remote_base_path, "object")
+
         self.remote_type_path            = os.path.join(self.remote_conf_path, "type")
         self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer")
 
         # Create directories
         self.__init_out_paths()
-
+        self.__init_remote_paths()
-        self.__init_env()
 
     def cleanup(self):
         # Do not use in __del__:
@@ -106,11 +105,6 @@ class Context:
             shutil.rmtree(self.cache_path)
         shutil.move(self.base_path, self.cache_path)
 
-    #def __init_env(self):
-    #    """Setup environment"""
-    #    os.environ['__cdist_out_path']   = self.out_path
-    #    os.environ['__cdist_base_path']  = self.base_path
-
     def __init_out_paths(self):
         """Initialise output directory structure"""
 
@@ -122,6 +116,13 @@ class Context:
         os.mkdir(self.global_explorer_out_path)
         os.mkdir(self.bin_path)
 
+    def __init_remote_paths(self):
+        """Initialise remote directory structure"""
+
+        self.remove_remote_path(self.remote_base_path)
+        self.remote_mkdir(self.remote_base_path)
+        self.remote_mkdir(self.remote_conf_path)
+
     def remote_mkdir(self, directory):
         """Create directory on remote side"""
         cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
@@ -129,3 +130,8 @@ class Context:
     def remove_remote_path(self, destination):
         cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=True)
 
+    def transfer_path(self, source, destination):
+        """Transfer directory and previously delete the remote destination"""
+        self.remove_remote_path(destination)
+        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
+            ["-r", source, self.target_host + ":" + destination])

lib/cdist/copy.py

@@ -1,13 +0,0 @@
-    # FIXME: To Copy
-    def transfer_dir(self, source, destination):
-        """Transfer directory and previously delete the remote destination"""
-        self.remove_remote_dir(destination)
-        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
-            ["-r", source, self.target_host + ":" + destination])
-
-    # FIXME: To Copy
-    def transfer_file(self, source, destination):
-        """Transfer file"""
-        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
-            [source, self.target_host + ":" + destination])
-

lib/cdist/emulator.py

@@ -29,9 +29,9 @@ log = logging.getLogger(__name__)
 
 def run(argv):
     """Emulate type commands (i.e. __file and co)"""
-    type            = os.path.basename(argv[0])
+    cdist_type      = os.path.basename(argv[0])
-    type_dir        = os.path.join(os.environ['__cdist_type_base_dir'], type)
+    type_path        = os.path.join(os.environ['__cdist_type_base_path'], cdist_type)
-    param_dir       = os.path.join(type_dir, "parameter")
+    param_dir       = os.path.join(type_path, "parameter")
     global_dir      = os.environ['__global']
     object_source   = os.environ['__cdist_manifest']
 
@@ -50,14 +50,14 @@ def run(argv):
         parser.add_argument(argument, action='store', required=True)
 
     # If not singleton support one positional parameter
-    if not os.path.isfile(os.path.join(type_dir, "singleton")):
+    if not os.path.isfile(os.path.join(type_path, "singleton")):
         parser.add_argument("object_id", nargs=1)
 
     # And finally verify parameter
     args = parser.parse_args(argv[1:])
 
     # Setup object_id
-    if os.path.isfile(os.path.join(type_dir, "singleton")):
+    if os.path.isfile(os.path.join(type_path, "singleton")):
         object_id = "singleton"
     else:
         object_id = args.object_id[0]
@@ -68,14 +68,14 @@ def run(argv):
             object_id = object_id[1:]
 
     # Prefix output by object_self
-    logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s'
+    logformat = '%(levelname)s: ' + cdist_type + '/' + object_id + ': %(message)s'
     logging.basicConfig(format=logformat)
 
     # FIXME: verify object id
     log.debug(args)
 
-    object_dir = os.path.join(global_dir, "object", type,
+    object_dir = os.path.join(global_dir, "object", cdist_type,
-                            object_id, cdist.path.DOT_CDIST)
+                            object_id, cdist.DOT_CDIST)
     log.debug("Object output dir = " + object_dir)
 
     param_out_dir = os.path.join(object_dir, "parameter")
@@ -132,7 +132,7 @@ def run(argv):
         requirements = os.environ['__require']
         log.debug(object_id + ":Writing requirements: " + requirements)
         require_fd = open(os.path.join(object_dir, "require"), "a")
-        require_fd.writelines(requirements.split(" "))
+        require_fd.write(requirements.replace(" ","\n"))
         require_fd.close()
 
     # Record / Append source
@@ -140,4 +140,4 @@ def run(argv):
     source_fd.writelines(object_source)
     source_fd.close()
 
-    log.debug("Finished " + type + "/" + object_id + repr(params))
+    log.debug("Finished " + cdist_type + "/" + object_id + repr(params))