Merge remote-tracking branch 'steven/master'

lib/cdist/core/code.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+import os
+
+import cdist
+
+log = logging.getLogger(__name__)
+
+
+'''
+common:
+    runs only locally, does not need remote
+
+    env:
+        PATH: prepend directory with type emulator symlinks == local.bin_path
+        __target_host: the target host we are working on
+        __cdist_manifest: full qualified path of the manifest == script
+        __cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator
+            == local.type_path
+
+gencode-local
+    script: full qualified path to a types gencode-local
+
+    env:
+        __target_host: the target host we are working on
+        __global: full qualified path to the global output dir == local.out_path
+        __object: full qualified path to the object's dir
+        __object_id: the objects id
+        __object_fq: full qualified object id, iow: $type.name + / + object_id
+        __type: full qualified path to the type's dir
+
+    returns: string containing the generated code or None
+
+gencode-remote
+    script: full qualified path to a types gencode-remote
+
+    env:
+        __target_host: the target host we are working on
+        __global: full qualified path to the global output dir == local.out_path
+        __object: full qualified path to the object's dir
+        __object_id: the objects id
+        __object_fq: full qualified object id, iow: $type.name + / + object_id
+        __type: full qualified path to the type's dir
+
+    returns: string containing the generated code or None
+
+
+code-local
+    script: full qualified path to object's code-local
+    - run script localy
+    returns: string containing the output
+
+code-remote
+    script: full qualified path to object's code-remote
+    - copy script to remote
+    - run script remotely
+    returns: string containing the output
+'''
+
+
+class Code(object):
+    """Generates and executes cdist code scripts.
+
+    """
+    def __init__(self, target_host, local, remote):
+        self.target_host = target_host
+        self.local = local
+        self.remote = remote
+        self.env = {
+            '__target_host': self.target_host,
+            '__global': self.local.out_path,
+        }
+
+    def _run_gencode(self, cdist_object, which):
+        cdist_type = cdist_object.type
+        script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which))
+        env = os.environ.copy()
+        env.update(self.env)
+        env.update({
+            '__type': cdist_object.type.absolute_path,
+            '__object': cdist_object.absolute_path,
+            '__object_id': cdist_object.object_id,
+            '__object_fq': cdist_object.path,
+        })
+        return self.local.run_script(script, env=env)
+
+    def run_gencode_local(self, cdist_object):
+        """Run the gencode-local script for the given cdist object."""
+        return self._run_gencode(cdist_object, 'local')
+
+    def run_gencode_remote(self, cdist_object):
+        """Run the gencode-remote script for the given cdist object."""
+        return self._run_gencode(cdist_object, 'remote')
+
+    def transfer_code_remote(self, cdist_object):
+        """Transfer the code_remote script for the given object to the remote side."""
+        source = os.path.join(self.local.object_path, cdist_object.code_remote_path)
+        destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path)
+        self.remote.mkdir(destination)
+        self.remote.transfer(source, destination)
+
+    def _run_code(self, cdist_object, which):
+        which_exec = getattr(self, which)
+        script = os.path.join(self.local.object_path, getattr(cdist_object, 'code_%s_path' % which))
+        return which_exec.run_script(script)
+
+    def run_code_local(self, cdist_object):
+        """Run the code-local script for the given cdist object."""
+        return self._run_code(cdist_object, 'local')
+
+    def run_code_remote(self, cdist_object):
+        """Run the code-remote script for the given cdist object on the remote side."""
+        return self._run_code(cdist_object, 'remote')

lib/cdist/core/explorer.py

@@ -74,7 +74,12 @@ class Explorer(object):
             '__explorer': self.remote.global_explorer_path,
         }
 
-    # FIXME: should i do this?
+    ### global
+
+    def list_global_explorer_names(self):
+        """Return a list of global explorer names."""
+        return os.listdir(self.local.global_explorer_path)
+
     def transfer_global_explorers(self):
         """Transfer the global explorers to the remote side."""
         self.remote.mkdir(self.remote.global_explorer_path)
@@ -85,7 +90,13 @@ class Explorer(object):
         script = os.path.join(self.remote.global_explorer_path, explorer)
         return self.remote.run_script(script, env=self.env)
 
-    # FIXME: should i do this?
+    ### type
+
+    def list_type_explorer_names(self, cdist_type):
+        """Return a list of explorer names for the given type."""
+        source = os.path.join(self.local.type_path, cdist_type.explorer_path)
+        return os.listdir(source)
+
     def transfer_type_explorers(self, cdist_type):
         """Transfer the type explorers for the given type to the remote side."""
         source = os.path.join(self.local.type_path, cdist_type.explorer_path)
@@ -93,9 +104,12 @@ class Explorer(object):
         self.remote.mkdir(destination)
         self.remote.transfer(source, destination)
 
-    # FIXME: should i do this? probably not
     def transfer_object_parameters(self, cdist_object):
-        pass
+        """Transfer the parameters for the given object to the remote side."""
+        source = os.path.join(self.local.object_path, cdist_object.parameter_path)
+        destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
+        self.remote.mkdir(destination)
+        self.remote.transfer(source, destination)
 
     def run_type_explorer(self, explorer, cdist_object):
         """Run the given type explorer for the given object and return it's output."""

lib/cdist/core/object.py

@@ -93,12 +93,14 @@ class Object(object):
         return os.path.join(self.path, "explorer")
 
     requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require'))
-    parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.absolute_path, 'parameter'))
+    parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path))
     explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path))
     changed = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "changed"))
     prepared = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "prepared"))
     ran = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "ran"))
     source = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, "source"))
+    code_local = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_local_path))
+    code_remote = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
 
     @property
     def exists(self):

lib/cdist/test/code/__init__.py

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import os
+import tempfile
+import unittest
+import shutil
+import getpass
+
+import cdist
+from cdist import core
+from cdist import test
+from cdist.exec import local
+from cdist.exec import remote
+from cdist.core import code
+
+import os.path as op
+my_dir = op.abspath(op.dirname(__file__))
+fixtures = op.join(my_dir, 'fixtures')
+local_base_path = fixtures
+
+class ExplorerClassTestCase(unittest.TestCase):
+
+    def mkdtemp(self, **kwargs):
+        return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs)
+
+    def mkstemp(self, **kwargs):
+        return tempfile.mkstemp(prefix='tmp.cdist.test.', **kwargs)
+
+    def setUp(self):
+        target_host = 'localhost'
+
+        self.local_base_path = local_base_path
+        self.out_path = self.mkdtemp()
+        self.local = local.Local(target_host, self.local_base_path, self.out_path)
+        self.local.create_directories()
+
+        self.remote_base_path = self.mkdtemp()
+        self.user = getpass.getuser()
+        remote_exec = "ssh -o User=%s -q" % self.user
+        remote_copy = "scp -o User=%s -q" % self.user
+        self.remote = remote.Remote(target_host, self.remote_base_path, remote_exec, remote_copy)
+
+        self.code = code.Code(target_host, self.local, self.remote)
+
+        self.cdist_type = core.Type(self.local.type_path, '__dump_environment')
+        self.cdist_object = core.Object(self.cdist_type, self.local.object_path, 'whatever')
+        self.cdist_object.create()
+
+    def tearDown(self):
+        shutil.rmtree(self.out_path)
+        shutil.rmtree(self.remote_base_path)
+
+    def test_run_gencode_local_environment(self):
+        output_string = self.code.run_gencode_local(self.cdist_object)
+        output_dict = {}
+        for line in output_string.split('\n'):
+            if line:
+                junk,value = line.split(': ')
+                key = junk.split(' ')[1]
+                output_dict[key] = value
+        self.assertEqual(output_dict['__target_host'], self.local.target_host)
+        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
+        self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
+        self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
+        self.assertEqual(output_dict['__object_fq'], self.cdist_object.path)
+
+    def test_run_gencode_remote_environment(self):
+        output_string = self.code.run_gencode_remote(self.cdist_object)
+        output_dict = {}
+        for line in output_string.split('\n'):
+            if line:
+                junk,value = line.split(': ')
+                key = junk.split(' ')[1]
+                output_dict[key] = value
+        self.assertEqual(output_dict['__target_host'], self.local.target_host)
+        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
+        self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
+        self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
+        self.assertEqual(output_dict['__object_fq'], self.cdist_object.path)
+
+    def test_transfer_code_remote(self):
+        self.cdist_object.code_remote = self.code.run_gencode_remote(self.cdist_object)
+        self.code.transfer_code_remote(self.cdist_object)
+        destination = os.path.join(self.remote.object_path, self.cdist_object.code_remote_path)
+        self.assertTrue(os.path.isfile(destination))
+
+    def test_run_code_local_environment(self):
+        self.cdist_object.code_local = self.code.run_gencode_local(self.cdist_object)
+        output_string = self.code.run_code_local(self.cdist_object)
+        output_dict = {}
+        for line in output_string.split('\n'):
+            if line:
+                key,value = line.split(': ')
+                output_dict[key] = value
+        self.assertEqual(output_dict['__target_host'], self.local.target_host)
+        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
+        self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
+        self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
+        self.assertEqual(output_dict['__object_fq'], self.cdist_object.path)
+        
+    def test_run_code_remote_environment(self):
+        self.cdist_object.code_remote = self.code.run_gencode_remote(self.cdist_object)
+        output_string = self.code.run_code_remote(self.cdist_object)
+        output_dict = {}
+        for line in output_string.split('\n'):
+            if line:
+                key,value = line.split(': ')
+                output_dict[key] = value
+        self.assertEqual(output_dict['__target_host'], self.local.target_host)
+        self.assertEqual(output_dict['__global'], self.local.out_path)
+        self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path)
+        self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path)
+        self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id)
+        self.assertEqual(output_dict['__object_fq'], self.cdist_object.path)
+
+'''
+    def test_list_type_explorer_names(self):
+        cdist_type = core.Type(self.local.type_path, '__test_type')
+        expected = cdist_type.explorers
+        self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected)
+
+    def test_transfer_type_explorers(self):
+        cdist_type = core.Type(self.local.type_path, '__test_type')
+        self.explorer.transfer_type_explorers(cdist_type)
+        source = os.path.join(self.local.type_path, cdist_type.explorer_path)
+        destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
+        self.assertEqual(os.listdir(source), os.listdir(destination))
+
+    def test_transfer_object_parameters(self):
+        cdist_type = core.Type(self.local.type_path, '__test_type')
+        cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
+        cdist_object.parameters = {'first': 'first value', 'second': 'second value'}
+        self.explorer.transfer_object_parameters(cdist_object)
+        source = os.path.join(self.local.object_path, cdist_object.parameter_path)
+        destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
+        self.assertEqual(os.listdir(source), os.listdir(destination))
+
+    def test_run_type_explorer(self):
+        cdist_type = core.Type(self.local.type_path, '__test_type')
+        cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
+        self.explorer.transfer_type_explorers(cdist_type)
+        output = self.explorer.run_type_explorer('world', cdist_object)
+        self.assertEqual(output, 'hello\n')
+'''

lib/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+echo "echo __target_host: $__target_host"
+echo "echo __global: $__global"
+echo "echo __type: $__type"
+echo "echo __object: $__object"
+echo "echo __object_id: $__object_id"
+echo "echo __object_fq: $__object_fq"

lib/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-remote

@@ -0,0 +1 @@
+gencode-local

lib/cdist/test/explorer/__init__.py

@@ -65,23 +65,46 @@ class ExplorerClassTestCase(unittest.TestCase):
         shutil.rmtree(self.out_path)
         shutil.rmtree(self.remote_base_path)
 
+    def test_list_global_explorer_names(self):
+        expected = ['global']
+        self.assertEqual(self.explorer.list_global_explorer_names(), expected)
+
     def test_transfer_global_explorers(self):
-        # FIXME: test result
         self.explorer.transfer_global_explorers()
+        source = self.local.global_explorer_path
+        destination = self.remote.global_explorer_path
+        self.assertEqual(os.listdir(source), os.listdir(destination))
 
     def test_run_global_explorer(self):
-        # FIXME: test result
         self.explorer.transfer_global_explorers()
-        self.explorer.run_global_explorer('global')
+        output = self.explorer.run_global_explorer('global')
+        self.assertEqual(output, 'global\n')
+
+    def test_list_type_explorer_names(self):
+        cdist_type = core.Type(self.local.type_path, '__test_type')
+        expected = cdist_type.explorers
+        self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected)
 
     def test_transfer_type_explorers(self):
-        # FIXME: test result
         cdist_type = core.Type(self.local.type_path, '__test_type')
         self.explorer.transfer_type_explorers(cdist_type)
+        source = os.path.join(self.local.type_path, cdist_type.explorer_path)
+        destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
+        self.assertEqual(os.listdir(source), os.listdir(destination))
+
+    def test_transfer_object_parameters(self):
+        cdist_type = core.Type(self.local.type_path, '__test_type')
+        cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
+        cdist_object.parameters = {'first': 'first value', 'second': 'second value'}
+        self.explorer.transfer_object_parameters(cdist_object)
+        source = os.path.join(self.local.object_path, cdist_object.parameter_path)
+        destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
+        self.assertEqual(os.listdir(source), os.listdir(destination))
 
     def test_run_type_explorer(self):
         cdist_type = core.Type(self.local.type_path, '__test_type')
         cdist_object = core.Object(cdist_type, self.local.object_path, 'whatever')
         self.explorer.transfer_type_explorers(cdist_type)
-        self.assertEqual(self.explorer.run_type_explorer('world', cdist_object), 'hello\n')
+        output = self.explorer.run_type_explorer('world', cdist_object)
+        self.assertEqual(output, 'hello\n')
 

lib/cdist/test/object/__init__.py

@@ -63,6 +63,8 @@ class ObjectTestCase(unittest.TestCase):
         self.cdist_object.prepared = False
         self.cdist_object.ran = False
         self.cdist_object.source = []
+        self.cdist_object.code_local = ''
+        self.cdist_object.code_remote = ''
 
     def test_name(self):
         self.assertEqual(self.cdist_object.name, '__third/moon')
@@ -136,3 +138,17 @@ class ObjectTestCase(unittest.TestCase):
     def test_source_after_changing(self):
         self.cdist_object.source = ['/path/to/manifest']
         self.assertEqual(list(self.cdist_object.source), ['/path/to/manifest'])
+
+    def test_code_local(self):
+        self.assertEqual(self.cdist_object.code_local, '')
+
+    def test_code_local_after_changing(self):
+        self.cdist_object.code_local = 'Hello World'
+        self.assertEqual(self.cdist_object.code_local, 'Hello World')
+
+    def test_code_remote(self):
+        self.assertEqual(self.cdist_object.code_remote, '')
+
+    def test_code_remote_after_changing(self):
+        self.cdist_object.code_remote = 'Hello World'
+        self.assertEqual(self.cdist_object.code_remote, 'Hello World')

lib/cdist/util/fsproperty.py

@@ -221,6 +221,8 @@ class DirectoryDictProperty(DirectoryDict):
     def __set__(self, obj, value):
         self._set_path(obj)
         if value is not None:
+            # create directory if it doesn't exist
+            os.makedirs(self.path, exist_ok=True)
             for name in self.keys():
                 del self[name]
             self.update(value)