Merge remote-tracking branch 'telmich/master'

Conflicts:
	lib/cdist/core/explorer.py
	lib/cdist/emulator.py

Signed-off-by: Steven Armstrong <steven@icarus.ethz.ch>

bin/cdist

@@ -145,6 +145,10 @@ def configinstall(args, mode):
     log.info("Total processing time for %s host(s): %s", len(args.host),
                 (time_end - time_start))
 
+def emulator():
+    """Prepare and run emulator"""
+    emulator = cdist.emulator.Emulator(sys.argv)
+    emulator.run()
 
 if __name__ == "__main__":
     try:
@@ -152,7 +156,8 @@ if __name__ == "__main__":
 
         if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
             import cdist.emulator
-            cdist.emulator.run(sys.argv)
+
+            emulator()
         else:
             import cdist.banner
             import cdist.config

lib/cdist/config_install.py

@@ -70,8 +70,8 @@ class ConfigInstall(object):
         start_time = time.time()
         self.deploy_to()
         self.cleanup()
-        self.log.info("Finished run of %s in %s seconds", 
+        self.log.info("Finished run in %s seconds", 
-            self.context.target_host, time.time() - start_time)
+            time.time() - start_time)
 
     def stage_prepare(self):
         """Do everything for a deploy, minus the actual code stage"""

lib/cdist/core/explorer.py

@@ -25,9 +25,6 @@ import os
 
 import cdist
 
-log = logging.getLogger(__name__)
-
-
 '''
 common:
     runs only remotely, needs local and remote to construct paths
@@ -67,13 +64,16 @@ class Explorer(object):
     """
     def __init__(self, target_host, local, remote):
         self.target_host = target_host
+
+        self.log = logging.getLogger(target_host)
+
         self.local = local
         self.remote = remote
         self.env = {
             '__target_host': self.target_host,
             '__explorer': self.remote.global_explorer_path,
         }
-        if log.getEffectiveLevel() == logging.DEBUG:
+        if self.log.getEffectiveLevel() == logging.DEBUG:
             self.env.update({'__debug': "yes" })
         self._type_explorers_transferred = []
 

lib/cdist/core/manifest.py

@@ -25,9 +25,6 @@ import os
 
 import cdist
 
-log = logging.getLogger(__name__)
-
-
 '''
 common:
     runs only locally, does not need remote
@@ -68,13 +65,16 @@ class Manifest(object):
     def __init__(self, target_host, local):
         self.target_host = target_host
         self.local = local
+        
+        self.log = logging.getLogger(self.target_host)
+
         self.env = {
             'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
             '__target_host': self.target_host,
             '__global': self.local.out_path,
             '__cdist_type_base_path': self.local.type_path, # for use in type emulator
         }
-        if log.getEffectiveLevel() == logging.DEBUG:
+        if self.log.getEffectiveLevel() == logging.DEBUG:
             self.env.update({'__debug': "yes" })
 
 
@@ -83,7 +83,7 @@ class Manifest(object):
         env.update(self.env)
         env['__manifest'] = self.local.manifest_path
         env['__cdist_manifest'] = script
-        log.info("Running initial manifest " + self.local.manifest_path)
+        self.log.info("Running initial manifest " + self.local.manifest_path)
         self.local.run_script(script, env=env)
 
     def run_type_manifest(self, cdist_object):

lib/cdist/emulator.py

@@ -26,8 +26,6 @@ import os
 import cdist
 from cdist import core
 
-log = logging.getLogger(__name__)
-
 
 class IllegalRequirementError(cdist.Error):
     def __init__(self, requirement, message=None):
@@ -37,102 +35,135 @@ class IllegalRequirementError(cdist.Error):
     def __str__(self):
         return '%s: %s' % (self.message, self.requirement)
 
+class Emulator(object):
+    def __init__(self, argv):
+        self.argv           = argv
+        self.object_id      = False
 
-def run(argv):
+        self.global_path    = os.environ['__global']
-    """Emulate type commands (i.e. __file and co)"""
+        self.object_source  = os.environ['__cdist_manifest']
-    global_path = os.environ['__global']
+        self.target_host    = os.environ['__target_host']
-    object_source = os.environ['__cdist_manifest']
+        self.type_base_path = os.environ['__cdist_type_base_path']
-    target_host = os.environ['__target_host']
+        
-    type_name = os.path.basename(argv[0])
+        self.object_base_path = os.path.join(self.global_path, "object")
 
-    # Logsetup - FIXME: add object_fq as soon as setup!
+        self.type_name      = os.path.basename(argv[0])
-    #id = target_host + ": " + cdist_type + '/' + object_id 
+        self.cdist_type     = core.Type(self.type_base_path, self.type_name)
-    id = target_host + ": "
-    # logformat = '%(levelname)s: ' + target_host + ": " + cdist_type + '/' + object_id + ': %(message)s'
-    logformat = '%(levelname)s: ' + id + ': %(message)s'
-    logging.basicConfig(format=logformat)
 
-    if '__debug' in os.environ:
+        self.__init_log()
-        logging.root.setLevel(logging.DEBUG)
-    else:
-        logging.root.setLevel(logging.INFO)
 
-    object_base_path = os.path.join(global_path, "object")
+    def filter(self, record):
-    type_base_path = os.environ['__cdist_type_base_path']
+        """Add hostname and object to logs via logging Filter"""
-    cdist_type = core.Type(type_base_path, type_name)
 
-    if '__install' in os.environ:
+        prefix = self.target_host + ": (emulator)"
-        if not cdist_type.is_install:
-            log.debug("Running in install mode, ignoring non install type")
-            return True
 
-    parser = argparse.ArgumentParser(add_help=False)
+        if self.object_id:
+            prefix = prefix + " " + self.type_name + "/" + self.object_id
 
-    for parameter in cdist_type.optional_parameters:
+        record.msg = prefix + ": " + record.msg
-        argument = "--" + parameter
-        parser.add_argument(argument, action='store', required=False)
-    for parameter in cdist_type.required_parameters:
-        argument = "--" + parameter
-        parser.add_argument(argument, action='store', required=True)
 
-    # If not singleton support one positional parameter
+        return True
-    if not cdist_type.is_singleton:
-        parser.add_argument("object_id", nargs=1)
 
-    # And finally verify parameter
+    def run(self):
-    args = parser.parse_args(argv[1:])
+        """Emulate type commands (i.e. __file and co)"""
 
-    # Setup object_id
+        if '__install' in os.environ:
-    if cdist_type.is_singleton:
+            if not self.cdist_type.is_install:
-        object_id = "singleton"
+                self.log.debug("Running in install mode, ignoring non install type")
-    else:
+                return True
-        object_id = args.object_id[0]
-        del args.object_id
 
-        # strip leading slash from object_id
+        self.commandline()
-        object_id = object_id.lstrip('/')
+        self.setup_object()
+        self.record_requirements()
+        self.log.debug("Finished %s %s" % (self.cdist_object.path, self.parameters))
 
-    # Instantiate the cdist object whe are defining
+    def __init_log(self):
-    cdist_object = core.Object(cdist_type, object_base_path, object_id)
+        """Setup logging facility"""
+        logformat = '%(levelname)s: %(message)s'
+        logging.basicConfig(format=logformat)
 
-    # FIXME: verify object id
+        if '__debug' in os.environ:
-    log.debug('#### emulator args: %s' % args)
+            logging.root.setLevel(logging.DEBUG)
+        else:
+            logging.root.setLevel(logging.INFO)
 
-    # Create object with given parameters
+        self.log            = logging.getLogger(__name__)
-    parameters = {}
+        self.log.addFilter(self)
-    for key,value in vars(args).items():
+
-        if value is not None:
+    def commandline(self):
-            parameters[key] = value
+        """Parse command line"""
-    
+
-    if cdist_object.exists:
+        parser = argparse.ArgumentParser(add_help=False)
-        if cdist_object.parameters != parameters:
+
-            raise cdist.Error("Object %s already exists with conflicting parameters:\n%s: %s\n%s: %s"
+        for parameter in self.cdist_type.optional_parameters:
-                % (cdist_object, " ".join(cdist_object.source), cdist_object.parameters, object_source, parameters)
+            argument = "--" + parameter
-            )
+            parser.add_argument(argument, action='store', required=False)
-    else:
+        for parameter in self.cdist_type.required_parameters:
-        cdist_object.create()
+            argument = "--" + parameter
-        cdist_object.parameters = parameters
+            parser.add_argument(argument, action='store', required=True)
+
+        # If not singleton support one positional parameter
+        if not self.cdist_type.is_singleton:
+            parser.add_argument("object_id", nargs=1)
+
+        # And finally parse/verify parameter
+        self.args = parser.parse_args(self.argv[1:])
+        self.log.debug('Args: %s' % self.args)
+
+
+    def setup_object(self):
+        # FIXME: verify object id
+
+        # Setup object_id
+        if self.cdist_type.is_singleton:
+            self.object_id = "singleton"
+        else:
+            self.object_id = self.args.object_id[0]
+            del self.args.object_id
 
-    # Record requirements
-    if "require" in os.environ:
-        requirements = os.environ['require']
-        for requirement in requirements.split(" "):
-            requirement_parts = requirement.split(os.sep, 1)
-            requirement_parts.reverse()
-            requirement_type_name = requirement_parts.pop()
-            try:
-                requirement_object_id = requirement_parts.pop()
-            except IndexError:
-                # no object id, must be singleton
-                requirement_object_id = 'singleton'
             # strip leading slash from object_id
-            if requirement_object_id.startswith('/'):
+            self.object_id = self.object_id.lstrip('/')
-                log.debug("Stripping leading slash from requirements object_id: %s", requirement)
+
+        # Instantiate the cdist object we are defining
+        self.cdist_object = core.Object(self.cdist_type, self.object_base_path, self.object_id)
+
+        # Create object with given parameters
+        self.parameters = {}
+        for key,value in vars(self.args).items():
+            if value is not None:
+                self.parameters[key] = value
+        
+        if self.cdist_object.exists:
+            if cdist_object.parameters != self.parameters:
+                raise cdist.Error("Object %s already exists with conflicting parameters:\n%s: %s\n%s: %s"
+                    % (self.cdist_object, " ".join(self.cdist_object.source), self.cdist_object.parameters, self.object_source, self.parameters)
+            )
+        else:
+            self.cdist_object.create()
+            self.cdist_object.parameters = self.parameters
+
+    def record_requirements(self):
+        """record requirements"""
+
+        if "require" in os.environ:
+            requirements = os.environ['require']
+            for requirement in requirements.split(" "):
+                self.log.debug("Recording requirement: " + requirement)
+                requirement_parts = requirement.split(os.sep, 1)
+                requirement_parts.reverse()
+                # FIXME: continue here
+                FAILHERE,PLEASE()[]!
+                print(requirement)
+                print(requirement_parts)
+                requirement_type_name = requirement_parts.pop()
+                try:
+                    requirement_object_id = requirement_parts.pop()
+                except IndexError:
+                    # no object id, must be singleton
+                    requirement_object_id = 'singleton'
+
                 requirement_object_id = requirement_object_id.lstrip('/')
-            log.debug("Recording requirement: %s -> %s" % (cdist_object.path, requirement))
+                self.cdist_object.requirements.append(requirement)
-            cdist_object.requirements.append(requirement)
 
-    # Record / Append source
+        # Record / Append source
-    cdist_object.source.append(object_source)
+        self.cdist_object.source.append(self.object_source)
-
-    log.debug("Finished %s %s" % (cdist_object.path, parameters))

lib/cdist/exec/remote.py

@@ -38,7 +38,15 @@ class RemoteScriptError(cdist.Error):
         self.script_content = script_content
 
     def __str__(self):
-        return "Remote script execution failed: %s %s" % (self.script, self.command)
+        plain_command = " ".join(self.command)
+        return "Remote script execution failed: %s" % plain_command
+
+class DecodeError(cdist.Error):
+    def __init__(self, command):
+        self.command = command
+
+    def __str__(self):
+        return "Cannot decode output of " + " ".join(self.command)
 
 
 class Remote(object):
@@ -121,6 +129,8 @@ class Remote(object):
             raise cdist.Error("Command failed: " + " ".join(command))
         except OSError as error:
             raise cdist.Error(" ".join(*args) + ": " + error.args[1])
+        except UnicodeDecodeError:
+            raise DecodeError(command)
 
     def run_script(self, script, env=None, return_output=False):
         """Run the given script with the given environment on the remote side.

other/tests_reintegrate/test_exec.py

@@ -1,79 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# 2011 Nico Schottelius (nico-cdist at schottelius.org)
-#
-# This file is part of cdist.
-#
-# cdist is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# cdist is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with cdist. If not, see <http://www.gnu.org/licenses/>.
-#
-#
-
-
-import os
-import shutil
-import tempfile
-import unittest
-
-import cdist.exec
-
-
-class ExecTestCase(unittest.TestCase):
-    def setUp(self):
-        """Create shell code and co."""
-
-        self.temp_dir = tempfile.mkdtemp()
-        self.shell_false = os.path.join(self.temp_dir, "shell_false")
-        self.shell_true  = os.path.join(self.temp_dir, "shell_true")
-
-        true_fd = open(self.shell_true, "w")
-        true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
-        true_fd.close()
-        
-        false_fd = open(self.shell_false, "w")
-        false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
-        false_fd.close()
-
-        target_host = "does.not.exist"
-        remote_exec = "ssh -o User=root -q"
-        remote_copy = "scp -o User=root -q"
-        self.wrapper = cdist.exec.Wrapper(target_host, remote_exec, remote_copy)
-
-    def tearDown(self):
-        shutil.rmtree(self.temp_dir)
-        
-    def test_local_success_shell(self):
-        try:
-            self.wrapper.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
-        except cdist.Error:
-            failed = True
-        else:
-            failed = False
-        self.assertFalse(failed)
-
-    def test_local_fail_shell(self):
-        self.assertRaises(cdist.Error, self.wrapper.shell_run_or_debug_fail,
-            self.shell_false, [self.shell_false])
-
-    def test_local_success(self):
-        try:
-            self.wrapper.run_or_fail(["/bin/true"])
-        except cdist.Error:
-            failed = True
-        else:
-            failed = False
-        self.assertFalse(failed)
-
-    def test_local_fail(self):
-        self.assertRaises(cdist.Error, self.wrapper.run_or_fail, ["/bin/false"])