Merge remote-tracking branch 'telmich/master'

doc/dev/todo/steven

@@ -0,0 +1,8 @@
+Object:
+   code_remote
+   code_local
+   gencode_local
+   gencode_remote
+
+Type:
+   explorer_path

lib/cdist/__init__.py

@@ -20,6 +20,7 @@
 #
 
 VERSION     = "2.0.3"
+DOT_CDIST   = ".cdist"
 
 class Error(Exception):
     """Base exception class for this project"""
@@ -34,3 +35,17 @@ class MissingEnvironmentVariableError(Error):
 
     def __str__(self):
         return 'Missing required environment variable: ' + str(self.name)
+
+def file_to_list(filename):
+    """Return list from \n seperated file"""
+    if os.path.isfile(filename):
+        file_fd = open(filename, "r")
+        lines = file_fd.readlines()
+        file_fd.close()
+
+        # Remove \n from all lines
+        lines = map(lambda s: s.strip(), lines)
+    else:
+        lines = []
+
+    return lines

lib/cdist/config_install.py

@@ -278,3 +278,39 @@ class ConfigInstall:
                     self.run_type_manifest(cdist_object)
                     cdist_object.prepared = True
                     new_objects_created = True
+
+    # FIXME Move into configinstall
+    def transfer_object_parameter(self, cdist_object):
+        """Transfer the object parameter to the remote destination"""
+        # Create base path before using mkdir -p
+        self.remote_mkdir(self.remote_object_parameter_dir(cdist_object))
+
+        # Synchronise parameter dir afterwards
+        self.transfer_dir(self.object_parameter_dir(cdist_object), 
+                                self.remote_object_parameter_dir(cdist_object))
+
+    # FIXME Move into configinstall
+    def transfer_global_explorers(self):
+        """Transfer the global explorers"""
+        self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR)
+        self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR)
+
+    # FIXME Move into configinstall
+    def transfer_type_explorers(self, type):
+        """Transfer explorers of a type, but only once"""
+        if type.transferred_explorers:
+            log.debug("Skipping retransfer for explorers of %s", type)
+            return
+        else:
+            # Do not retransfer
+            type.transferred_explorers = True
+
+        # FIXME: Can be explorer_path or explorer_dir, I don't care.
+        src = type.explorer_path()
+        dst = type.remote_explorer_path()
+
+        # Transfer if there is at least one explorer
+        if len(type.explorers) > 0:
+            # Ensure that the path exists
+            self.remote_mkdir(dst)
+            self.transfer_dir(src, dst)

lib/cdist/context.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+import os
+import shutil
+import sys
+import tempfile
+
+
+log = logging.getLogger(__name__)
+
+import cdist.exec
+
+class Path:
+    """Class that handles path related configurations"""
+
+    def __init__(self, target_host, initial_manifest=False, debug=False):
+
+        self.target_host = target_host
+
+        # Base and Temp Base 
+        if "__cdist_base_dir" in os.environ:
+            self.base_dir = os.environ['__cdist_base_dir']
+        else:
+            self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
+
+        # Input directories
+        self.conf_dir               = os.path.join(self.base_dir, "conf")
+        self.cache_base_dir         = os.path.join(self.base_dir, "cache")
+        self.cache_dir              = os.path.join(self.cache_base_dir, target_host)
+        self.global_explorer_dir    = os.path.join(self.conf_dir, "explorer")
+        self.lib_dir                = os.path.join(self.base_dir, "lib")
+        self.manifest_dir           = os.path.join(self.conf_dir, "manifest")
+        self.type_base_dir          = os.path.join(self.conf_dir, "type")
+
+        # Mostly static, but can be overwritten on user demand
+        if initial_manifest:
+            self.initial_manifest = initial_manifest
+        else:
+            self.initial_manifest = os.path.join(self.manifest_dir, "init")
+
+        # Output directories
+        if "__cdist_out_dir" in os.environ:
+            self.out_dir = os.environ['__cdist_out_dir']
+        else:
+            self.out_dir = os.path.join(tempfile.mkdtemp(), "out")
+
+        self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
+        self.object_base_dir = os.path.join(self.out_dir, "object")
+        self.bin_dir = os.path.join(self.out_dir, "bin")
+
+        # Remote directories
+        if "__cdist_remote_base_dir" in os.environ:
+            self.remote_base_dir = os.environ['__cdist_remote_base_dir']
+        else:
+            self.remote_base_dir = "/var/lib/cdist"
+
+        self.remote_conf_dir            = os.path.join(self.remote_base_dir, "conf")
+        self.remote_object_dir          = os.path.join(self.remote_base_dir, "object")
+        self.remote_type_dir            = os.path.join(self.remote_conf_dir, "type")
+        self.remote_global_explorer_dir = os.path.join(self.remote_conf_dir, "explorer")
+
+        # Create directories
+        self.__init_out_dirs()
+
+    def cleanup(self):
+        # Do not use in __del__:
+        # http://docs.python.org/reference/datamodel.html#customization
+        # "other globals referenced by the __del__() method may already have been deleted 
+        # or in the process of being torn down (e.g. the import machinery shutting down)"
+        #
+        log.debug("Saving" + self.base_dir + "to " + self.cache_dir)
+        # Remove previous cache
+        if os.path.exists(self.cache_dir):
+            shutil.rmtree(self.cache_dir)
+        shutil.move(self.base_dir, self.cache_dir)
+
+    def __init_env(self):
+        """Setup environment"""
+        os.environ['__cdist_out_dir'] = self.out_dir
+
+    def __init_out_dirs(self):
+        """Initialise output directory structure"""
+
+        # Create base dir, if user supplied and not existing
+        if not os.isdir(self.base_dir):
+            os.mkdir(self.base_dir)
+            
+        os.mkdir(self.out_dir)
+        os.mkdir(self.global_explorer_out_dir)
+        os.mkdir(self.bin_dir)
+
+    def remote_mkdir(self, directory):
+        """Create directory on remote side"""
+        cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
+
+    def remove_remote_dir(self, destination):
+        cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=True)
+

lib/cdist/copy.py

@@ -0,0 +1,13 @@
+    # FIXME: To Copy
+    def transfer_dir(self, source, destination):
+        """Transfer directory and previously delete the remote destination"""
+        self.remove_remote_dir(destination)
+        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
+            ["-r", source, self.target_host + ":" + destination])
+
+    # FIXME: To Copy
+    def transfer_file(self, source, destination):
+        """Transfer file"""
+        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
+            [source, self.target_host + ":" + destination])
+

lib/cdist/core/globalexplorer.py

@@ -0,0 +1,66 @@
+    # FIXME: Explorer or stays
+    def global_explorer_output_path(self, explorer):
+        """Returns path of the output for a global explorer"""
+        return os.path.join(self.global_explorer_out_dir, explorer)
+
+    # FIXME Stays here / Explorer?
+    def remote_global_explorer_path(self, explorer):
+        """Returns path to the remote explorer"""
+        return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)
+
+    # FIXME: stays here
+    def list_global_explorers(self):
+        """Return list of available explorers"""
+        return os.listdir(self.global_explorer_dir)
+
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+import os
+log = logging.getLogger(__name__)
+
+class Explorer(object):
+
+    def __init__(self, src_dir, dst_dir):
+        self.src_dir = src_dir
+        self.dst_dir = dst_dir
+
+    def list_explorers(self):
+        """Return list of available explorers"""
+        dir = os.path.join(self.path, "explorer")
+        if os.path.isdir(dir):
+            list = os.listdir(dir)
+        else:
+            list = []
+
+        log.debug("Explorers for %s in %s: %s", type, dir, list)
+
+        return list
+
+    def is_install(self):
+        """Check whether a type is used for installation (if not: for configuration)"""
+        return os.path.isfile(os.path.join(self.path, "install"))
+
+    def remote_explorer_dir(self):
+        """Return remote directory that holds the explorers of a type"""
+        return os.path.join(self.remote_path, "explorer")

lib/cdist/core/object.py

@@ -162,3 +162,74 @@ class Object(object):
     ### /changed
 
     # FIXME: implement other properties/methods 
+
+    # FIXME: check following methods: implement or revoke / delete
+    # FIXME: Object
+    def get_object_id_from_object(self, cdist_object):
+        """Returns everything but the first part (i.e. object_id) of an object"""
+        return os.sep.join(cdist_object.split(os.sep)[1:])
+
+    # FIXME: Object
+    def object_dir(self, cdist_object):
+        """Returns the full path to the object (including .cdist)"""
+        return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)
+
+    # FIXME: Object
+    def remote_object_dir(self, cdist_object):
+        """Returns the remote full path to the object (including .cdist)"""
+        return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)
+
+    # FIXME: Object
+    def object_parameter_dir(self, cdist_object):
+        """Returns the dir to the object parameter"""
+        return os.path.join(self.object_dir(cdist_object), "parameter")
+
+    # FIXME: object
+    def remote_object_parameter_dir(self, cdist_object):
+        """Returns the remote dir to the object parameter"""
+        return os.path.join(self.remote_object_dir(cdist_object), "parameter")
+
+    # FIXME: object
+    def object_code_paths(self, cdist_object):
+        """Return paths to code scripts of object"""
+        return [os.path.join(self.object_dir(cdist_object), "code-local"),
+                  os.path.join(self.object_dir(cdist_object), "code-remote")]
+
+    # Stays here
+    def list_object_paths(self, starting_point):
+        """Return list of paths of existing objects"""
+        object_paths = []
+        
+        for content in os.listdir(starting_point):
+            full_path = os.path.join(starting_point, content)
+            if os.path.isdir(full_path):
+                object_paths.extend(self.list_object_paths(starting_point = full_path))
+                
+            # Directory contains .cdist -> is an object
+            if content == DOT_CDIST:
+                object_paths.append(starting_point)
+                
+        return object_paths
+
+    # Stays here
+    def list_objects(self):
+        """Return list of existing objects"""
+        
+        objects = []
+        if os.path.isdir(self.object_base_dir):
+            object_paths = self.list_object_paths(self.object_base_dir)
+            
+            for path in object_paths:
+                objects.append(os.path.relpath(path, self.object_base_dir))
+                
+        return objects
+
+    # FIXME: object
+    def type_explorer_output_dir(self, cdist_object):
+        """Returns and creates dir of the output for a type explorer"""
+        dir = os.path.join(self.object_dir(cdist_object), "explorer")
+        if not os.path.isdir(dir):
+            os.mkdir(dir)
+
+        return dir
+

lib/cdist/core/type.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 #
-# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -68,6 +69,8 @@ class Type(object):
         self.__required_parameters = None
         self.__optional_parameters = None
 
+        self.transferred_explorers = False
+
     def __repr__(self):
         return '<Type name=%s>' % self.name
 

lib/cdist/emulator.py

@@ -43,10 +43,10 @@ def run(argv):
 
     parser = argparse.ArgumentParser(add_help=False)
 
-    for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")):
+    for parameter in cdist.file_to_list(os.path.join(param_dir, "optional")):
         argument = "--" + parameter
         parser.add_argument(argument, action='store', required=False)
-    for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")):
+    for parameter in cdist.file_to_list(os.path.join(param_dir, "required")):
         argument = "--" + parameter
         parser.add_argument(argument, action='store', required=True)
 

lib/cdist/path.py

@@ -1,273 +0,0 @@
-# -*- coding: utf-8 -*-
-#
-# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-import logging
-import os
-import shutil
-import sys
-import tempfile
-
-# Hardcoded paths usually not changable
-REMOTE_BASE_DIR                 = "/var/lib/cdist"
-REMOTE_CONF_DIR                 = os.path.join(REMOTE_BASE_DIR, "conf")
-REMOTE_OBJECT_DIR               = os.path.join(REMOTE_BASE_DIR, "object")
-REMOTE_TYPE_DIR                 = os.path.join(REMOTE_CONF_DIR, "type")
-REMOTE_GLOBAL_EXPLORER_DIR      = os.path.join(REMOTE_CONF_DIR, "explorer")
-
-DOT_CDIST                       = ".cdist"
-
-log = logging.getLogger(__name__)
-
-import cdist.exec
-
-def file_to_list(filename):
-    """Return list from \n seperated file"""
-    if os.path.isfile(filename):
-        file_fd = open(filename, "r")
-        lines = file_fd.readlines()
-        file_fd.close()
-
-        # Remove \n from all lines
-        lines = map(lambda s: s.strip(), lines)
-    else:
-        lines = []
-
-    return lines
-
-class Path:
-    """Class that handles path related configurations"""
-
-    def __init__(self,
-                target_host,
-                initial_manifest=False,
-                base_dir=None,
-                debug=False):
-
-        # Base and Temp Base 
-        if base_dir:
-            self.base_dir = base_dir
-        else:
-            self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))
-
-        self.temp_dir = tempfile.mkdtemp()
-
-        self.target_host = target_host
-
-        # Input directories
-        self.conf_dir               = os.path.join(self.base_dir, "conf")
-        self.cache_base_dir         = os.path.join(self.base_dir, "cache")
-        self.cache_dir              = os.path.join(self.cache_base_dir, target_host)
-        self.global_explorer_dir    = os.path.join(self.conf_dir, "explorer")
-        self.lib_dir                = os.path.join(self.base_dir, "lib")
-        self.manifest_dir           = os.path.join(self.conf_dir, "manifest")
-        self.type_base_dir          = os.path.join(self.conf_dir, "type")
-
-        # Mostly static, but can be overwritten on user demand
-        if initial_manifest:
-            self.initial_manifest = initial_manifest
-        else:
-            self.initial_manifest = os.path.join(self.manifest_dir, "init")
-
-        # Output directories
-        self.out_dir = os.path.join(self.temp_dir, "out")
-        self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer")
-        self.object_base_dir = os.path.join(self.out_dir, "object")
-        self.bin_dir = os.path.join(self.out_dir, "bin")
-
-        os.environ['__cdist_out_dir'] = self.out_dir
-
-        # List of type explorers transferred
-        self.type_explorers_transferred = {}
-
-        # objects prepared
-        self.objects_prepared = []
-
-        # Create directories
-        self.__init_out_dirs()
-
-    def cleanup(self):
-        # Do not use in __del__:
-        # http://docs.python.org/reference/datamodel.html#customization
-        # "other globals referenced by the __del__() method may already have been deleted 
-        # or in the process of being torn down (e.g. the import machinery shutting down)"
-        #
-        log.debug("Saving" + self.temp_dir + "to " + self.cache_dir)
-        # Remove previous cache
-        if os.path.exists(self.cache_dir):
-            shutil.rmtree(self.cache_dir)
-        shutil.move(self.temp_dir, self.cache_dir)
-
-
-    def __init_out_dirs(self):
-        """Initialise output directory structure"""
-        os.mkdir(self.out_dir)
-        os.mkdir(self.global_explorer_out_dir)
-        os.mkdir(self.bin_dir)
-
-
-    # Stays here
-    def list_types(self):
-        """Retuns list of types"""
-        return os.listdir(self.type_base_dir)
-
-    ###################################################################### 
-
-    # FIXME: belongs to here - clearify remote*
-    def remote_mkdir(self, directory):
-        """Create directory on remote side"""
-        cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True)
-
-    # FIXME: belongs to here - clearify remote*
-    def remove_remote_dir(self, destination):
-        cdist.exec.run_or_fail(["rm", "-rf",  destination], remote_prefix=True)
-
-    # FIXME: belongs to here - clearify remote*
-    def transfer_dir(self, source, destination):
-        """Transfer directory and previously delete the remote destination"""
-        self.remove_remote_dir(destination)
-        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
-            ["-r", source, self.target_host + ":" + destination])
-
-    # FIXME: belongs to here - clearify remote*
-    def transfer_file(self, source, destination):
-        """Transfer file"""
-        cdist.exec.run_or_fail(os.environ['__remote_copy'].split() +
-            [source, self.target_host + ":" + destination])
-
-    # FIXME: Explorer or stays
-    def global_explorer_output_path(self, explorer):
-        """Returns path of the output for a global explorer"""
-        return os.path.join(self.global_explorer_out_dir, explorer)
-
-    # FIXME: object
-    def type_explorer_output_dir(self, cdist_object):
-        """Returns and creates dir of the output for a type explorer"""
-        dir = os.path.join(self.object_dir(cdist_object), "explorer")
-        if not os.path.isdir(dir):
-            os.mkdir(dir)
-
-        return dir
-
-    # FIXME Stays here / Explorer?
-    def remote_global_explorer_path(self, explorer):
-        """Returns path to the remote explorer"""
-        return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer)
-
-    # FIXME: stays here
-    def list_global_explorers(self):
-        """Return list of available explorers"""
-        return os.listdir(self.global_explorer_dir)
-
-    # Stays here
-    def list_object_paths(self, starting_point):
-        """Return list of paths of existing objects"""
-        object_paths = []
-
-        for content in os.listdir(starting_point):
-            full_path = os.path.join(starting_point, content)
-            if os.path.isdir(full_path):
-                object_paths.extend(self.list_object_paths(starting_point = full_path))
-
-            # Directory contains .cdist -> is an object
-            if content == DOT_CDIST:
-                object_paths.append(starting_point)
-
-        return object_paths
-
-    # FIXME: Object
-    def get_object_id_from_object(self, cdist_object):
-        """Returns everything but the first part (i.e. object_id) of an object"""
-        return os.sep.join(cdist_object.split(os.sep)[1:])
-
-    # FIXME: Object
-    def object_dir(self, cdist_object):
-        """Returns the full path to the object (including .cdist)"""
-        return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST)
-
-    # FIXME: Object
-    def remote_object_dir(self, cdist_object):
-        """Returns the remote full path to the object (including .cdist)"""
-        return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST)
-
-    # FIXME: Object
-    def object_parameter_dir(self, cdist_object):
-        """Returns the dir to the object parameter"""
-        return os.path.join(self.object_dir(cdist_object), "parameter")
-
-    # FIXME: object
-    def remote_object_parameter_dir(self, cdist_object):
-        """Returns the remote dir to the object parameter"""
-        return os.path.join(self.remote_object_dir(cdist_object), "parameter")
-
-    # FIXME: object
-    def object_code_paths(self, cdist_object):
-        """Return paths to code scripts of object"""
-        return [os.path.join(self.object_dir(cdist_object), "code-local"),
-                  os.path.join(self.object_dir(cdist_object), "code-remote")]
-
-    # Stays here
-    def list_objects(self):
-        """Return list of existing objects"""
-
-        objects = []
-        if os.path.isdir(self.object_base_dir):
-            object_paths = self.list_object_paths(self.object_base_dir)
-
-            for path in object_paths:
-                objects.append(os.path.relpath(path, self.object_base_dir))
-
-        return objects
-
-    # Stays here
-    def transfer_object_parameter(self, cdist_object):
-        """Transfer the object parameter to the remote destination"""
-        # Create base path before using mkdir -p
-        self.remote_mkdir(self.remote_object_parameter_dir(cdist_object))
-
-        # Synchronise parameter dir afterwards
-        self.transfer_dir(self.object_parameter_dir(cdist_object), 
-                                self.remote_object_parameter_dir(cdist_object))
-
-    # Stays here
-    def transfer_global_explorers(self):
-        """Transfer the global explorers"""
-        self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR)
-        self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR)
-
-    # Stays here - FIXME: adjust to type code, loop over types!
-    def transfer_type_explorers(self, type):
-        """Transfer explorers of a type, but only once"""
-        if type.transferred:
-            log.debug("Skipping retransfer for explorers of %s", type)
-            return
-        else:
-            # Do not retransfer
-            type.transferred = True
-
-        # FIXME: Can be explorer_path or explorer_dir, I don't care.
-        src = type.explorer_path()
-        dst = type.remote_explorer_path()
-
-        # Transfer if there is at least one explorer
-        if len(type.explorers) > 0:
-            # Ensure that the path exists
-            self.remote_mkdir(dst)
-            self.transfer_dir(src, dst)