Merge remote-tracking branch 'telmich/master'

README

@@ -15,7 +15,7 @@
        "P'        ""         ""  
      
 
-[[!toc levels=2]]
+[[!toc levels=3]]
 
 ## Introduction
 
@@ -86,13 +86,46 @@ cdist was tested or is know to run on at least
  * SSH-Server
 
 
-## Getting cdist
+## Installation
+
+### Preperation
+
+Ensure you have Python 3.x and the **argparse** module installed on
+the machine you use to **deploy to the targets**.
+
+#### Archlinux
+
+Archlinux already has python >= 3.2, so you only need to do:
+
+    pacman -S python
+
+#### Debian
+
+    aptitude install python3 python3-setuptools
+    easy_install3 argparse
+
+
+#### Gentoo
+
+Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3).
+If you want to ensure nothing breaks you must set back the python version to what was default before.
+
+    emerge -av =python-3.2.2 --autounmask-write
+    emerge -av =python-3.2.2
+    eselect python list
+    eselect python list set python3.2
+
+#### Max OS X
+
+Ensure you have port installed and configured (http://www.macports.org/install.php).
+
+    port install python32
+    ln -s /opt/local/bin/python3.2 /opt/local/bin/python3
+
+### Get cdist
 
 You can clone cdist from git, which gives you the advantage of having
 a version control in place for development of your own stuff as well.
-
-### Installation
-
 To install cdist, execute the following commands:
 
     git clone git://git.schottelius.org/cdist

bin/cdist

@@ -26,7 +26,7 @@ import os
 import re
 import sys
 
-log = logging.getLogger(__name__)
+log = logging.getLogger("cdist")
 
 # Ensure our /lib/ is included into PYTHON_PATH
 sys.path.insert(0, os.path.abspath(
@@ -39,12 +39,17 @@ def commandline():
     # Construct parser others can reuse
     parser = {}
     # Options _all_ parsers have in common
-    parser['most'] = argparse.ArgumentParser(add_help=False)
+    parser['loglevel'] = argparse.ArgumentParser(add_help=False)
-    parser['most'].add_argument('-d', '--debug',
+    parser['loglevel'].add_argument('-d', '--debug',
-        help='Set log level to debug', action='store_true')
+        help='Set log level to debug', action='store_true',
+        default=False)
+    parser['loglevel'].add_argument('-v', '--verbose',
+        help='Set log level to info, be more verbose',
+        action='store_true', default=False)
 
     # Main subcommand parser
-    parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION)
+    parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
+        parents=[parser['loglevel']])
     parser['main'].add_argument('-V', '--version',
         help='Show version', action='version',
         version='%(prog)s ' + cdist.VERSION)
@@ -52,7 +57,7 @@ def commandline():
 
     # Banner
     parser['banner'] = parser['sub'].add_parser('banner', 
-        add_help=False)
+        parents=[parser['loglevel']])
     parser['banner'].set_defaults(func=cdist.banner.banner)
 
     # Config and install (common stuff)
@@ -74,12 +79,12 @@ def commandline():
 
     # Config
     parser['config'] = parser['sub'].add_parser('config',
-        parents=[parser['most'], parser['configinstall']])
+        parents=[parser['loglevel'], parser['configinstall']])
     parser['config'].set_defaults(func=cdist.config.config)
 
     # Install
     parser['install'] = parser['sub'].add_parser('install',
-        parents=[parser['most'], parser['configinstall']])
+        parents=[parser['loglevel'], parser['configinstall']])
     parser['install'].set_defaults(func=cdist.install.install)
 
     for p in parser:
@@ -87,17 +92,18 @@ def commandline():
 
     args = parser['main'].parse_args(sys.argv[1:])
 
-    # Most subcommands have --debug, so handle it here
+    # Loglevels are handled globally in here and debug wins over verbose
-    if 'debug' in args:
+    if args.verbose:
+        logging.root.setLevel(logging.INFO)
     if args.debug:
         logging.root.setLevel(logging.DEBUG)
-    log.debug(args)
 
+    log.debug(args)
     args.func(args)
 
 if __name__ == "__main__":
     try:
-        logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+        logging.basicConfig(format='%(levelname)s: %(message)s')
 
         if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])):
             import cdist.emulator

build.sh

@@ -126,6 +126,14 @@ case "$1" in
       | xargs rm -f
    ;;
 
+   test)
+      python3 -m unittest discover test 'test_*.py' 
+   ;;
+
+   test-all)
+      python3 -m unittest discover test '*.py' 
+   ;;
+
    *)
       echo ''
       echo 'Welcome to cdist!'

doc/changelog

@@ -1,5 +1,10 @@
-2.0.2:
+2.0.3:
+	* Improved logging, added --verbose, by more quiet by default
+
+2.0.2: 2011-09-27
 	* Add support for detection of OpenWall Linux (Matthias Teege)
+	* Add support for __debug variable in manifests
+	* Bugfix core: Various issues with type emulator
 
 2.0.1: 2011-09-23
 	* Bugfix core: Always print source of error in case of exec errors

doc/dev/todo/TAKEME

@@ -34,6 +34,8 @@ USER INTERFACE
    -> given after manifest run already!
 
 - use absent/present for state by default?
+- buggy output with packages that don't exist in archlinux and fedora:
+   python3 vs. python
 
 TYPES
 ------

doc/dev/todo/niconext

@@ -311,15 +311,13 @@ eof
             via __global/
 
 - Support parallel execution
-   - and maximum number of parallel runs (-p X)
    - error handling / report failed hosts
 
-- Allow manifest to be read from stdin
 - Create new video for cdist 2.0.0
    http://www.youtube.com/watch?v=PRMjzy48eTI
 
 - Setup __debug, if -d is given, so other tools can reuse it
-   (-> non core feature!
+   - implement everywhere to external!
 
 - remote_prefix:
    scp vs. ssh issue

doc/man/cdist-reference.text.sh

@@ -166,6 +166,11 @@ changed::
 
 ENVIRONMENT VARIABLES
 ---------------------
+__debug::
+   If this variable is setup, cdist runs in debug mode.
+   You can use this information, to only output stuff in debug
+   mode as well.
+   Available for: initial manifest, type manifest
 __explorer::
    Directory that contains all global explorers.
    Available for: explorer

lib/cdist/__init__.py

@@ -19,7 +19,7 @@
 #
 #
 
-VERSION     = "2.0.2"
+VERSION     = "2.0.3"
 
 class Error(Exception):
     """Base exception class for this project"""

lib/cdist/config.py

@@ -65,6 +65,7 @@ class Config:
 
     def run_global_explores(self):
         """Run global explorers"""
+        log.info("Running global explorers")
         explorers = self.path.list_global_explorers()
         if(len(explorers) == 0):
             raise CdistError("No explorers found in", self.path.global_explorer_dir)
@@ -107,18 +108,22 @@ class Config:
             cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix)
             output_fd.close()
 
+    def link_emulator(self):
+        """Link emulator to types"""
+        cdist.emulator.link(self.exec_path,
+            self.path.bin_dir, self.path.list_types())
+
     def init_deploy(self):
         """Ensure the base directories are cleaned up"""
         log.debug("Creating clean directory structure")
 
         self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR)
         self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR)
-
+        self.link_emulator()
-        cdist.emulator.link(self.exec_path,
-            self.path.bin_dir, self.path.list_types())
 
     def run_initial_manifest(self):
         """Run the initial manifest"""
+        log.info("Running initial manifest %s", self.path.initial_manifest)
         env = {  "__manifest" : self.path.manifest_dir }
         self.run_manifest(self.path.initial_manifest, extra_env=env)
 
@@ -146,6 +151,10 @@ class Config:
         env['__target_host']            = self.target_host
         env['__global']                 = self.path.out_dir
         
+        # Submit debug flag to manifest, can be used by emulator and types
+        if self.debug:
+            env['__debug']                  = "yes"
+
         # Required for recording source
         env['__cdist_manifest']         = manifest
 
@@ -235,12 +244,13 @@ class Config:
         self.run_global_explores()
         self.run_initial_manifest()
         
+        log.info("Running object manifests and type explorers")
+
         old_objects = []
         objects = self.path.list_objects()
 
         # Continue process until no new objects are created anymore
         while old_objects != objects:
-            log.debug("Prepare stage")
             old_objects = list(objects)
             for cdist_object in objects:
                 if cdist_object in self.objects_prepared:
@@ -255,7 +265,7 @@ class Config:
 
     def stage_run(self):
         """The final (and real) step of deployment"""
-        log.debug("Actual run objects")
+        log.info("Generating and executing code")
         # Now do the final steps over the existing objects
         for cdist_object in self.path.list_objects():
             log.debug("Run object: %s", cdist_object)
@@ -298,7 +308,7 @@ def config(args):
 
     if args.parallel:
         for p in process.keys():
-            log.debug("Joining %s", p)
+            log.debug("Joining process %s", p)
             process[p].join()
 
     time_end = datetime.datetime.now()

lib/cdist/emulator.py

@@ -36,20 +36,21 @@ def run(argv):
     global_dir      = os.environ['__global']
     object_source   = os.environ['__cdist_manifest']
 
+    if '__debug' in os.environ:
+        logging.root.setLevel(logging.DEBUG)
+    else:
+        logging.basicConfig(level=logging.INFO)
+
     parser = argparse.ArgumentParser(add_help=False)
 
-    # Setup optional parameters
     for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")):
         argument = "--" + parameter
         parser.add_argument(argument, action='store', required=False)
-
-    # Setup required parameters
     for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")):
         argument = "--" + parameter
         parser.add_argument(argument, action='store', required=True)
 
-    # Setup positional parameter, if not singleton
+    # If not singleton support one positional parameter
-
     if not os.path.isfile(os.path.join(type_dir, "singleton")):
         parser.add_argument("object_id", nargs=1)
 
@@ -67,6 +68,10 @@ def run(argv):
         if object_id[0] == '/':
             object_id = object_id[1:]
 
+    # Prefix output by object_self
+    logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s'
+    logging.basicConfig(format=logformat)
+
     # FIXME: verify object id
     log.debug(args)
 
@@ -110,12 +115,12 @@ def run(argv):
                     value_old = param_fd.readlines()
                     param_fd.close()
                     
-                    if(value_old != value):
+                    if(value_old[0] != value):
-                        raise cdist.Error("Parameter + \"" + param +
+                        raise cdist.Error("Parameter\"" + param +
                             "\" differs: " + " ".join(value_old) + " vs. " +
                             value +
                             "\nSource = " + " ".join(old_object_source)
-                            + " new =" + object_source)
+                            + " new = " + object_source)
             else:
                 param_fd = open(file, "w")
                 param_fd.writelines(value)
@@ -124,7 +129,7 @@ def run(argv):
     # Record requirements
     if "__require" in os.environ:
         requirements = os.environ['__require']
-        print(object_id + ":Writing requirements: " + requirements)
+        log.debug(object_id + ":Writing requirements: " + requirements)
         require_fd = open(os.path.join(object_dir, "require"), "a")
         require_fd.writelines(requirements.split(" "))
         require_fd.close()
@@ -134,7 +139,7 @@ def run(argv):
     source_fd.writelines(object_source)
     source_fd.close()
 
-    print("Finished " + type + "/" + object_id + repr(params))
+    log.debug("Finished " + type + "/" + object_id + repr(params))
 
 
 def link(exec_path, bin_dir, type_list):

test/nico_ui.py

@@ -0,0 +1,45 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+import os
+import subprocess
+import unittest
+
+cdist_commands=["banner", "config", "install"]
+
+cdist_exec_path = os.path.abspath(
+    os.path.join(os.path.dirname(os.path.realpath(__file__)), "../bin/cdist"))
+
+class UI(unittest.TestCase):
+    def test_banner(self):
+        self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0)
+
+    def test_help(self):
+        for cmd in cdist_commands:
+            self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0)
+
+    # FIXME: mockup needed
+    def test_config_localhost(self):
+        for cmd in cdist_commands:
+            self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0)
+

test/test_config.py

@@ -20,68 +20,19 @@
 #
 #
 
-
 import os
 import sys
-import shutil
 import tempfile
 import unittest
 
 sys.path.insert(0, os.path.abspath(
-        os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib')))
+        os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
+
+import cdist.config
 
 cdist_exec_path = os.path.abspath(
     os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist"))
 
-import cdist
-import cdist.config
-import cdist.exec
-
-class Exec(unittest.TestCase):
-    def setUp(self):
-        """Create shell code and co."""
-
-        self.temp_dir = tempfile.mkdtemp()
-        self.shell_false = os.path.join(self.temp_dir, "shell_false")
-        self.shell_true  = os.path.join(self.temp_dir, "shell_true")
-
-        true_fd = open(self.shell_true, "w")
-        true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
-        true_fd.close()
-        
-        false_fd = open(self.shell_false, "w")
-        false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
-        false_fd.close()
-
-    def tearDown(self):
-        shutil.rmtree(self.temp_dir)
-        
-    def test_local_success_shell(self):
-        try:
-            cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
-        except cdist.Error:
-            failed = True
-        else:
-            failed = False
-
-        self.assertFalse(failed)
-
-    def test_local_fail_shell(self):
-        self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
-            self.shell_false, [self.shell_false])
-
-    def test_local_success(self):
-        try:
-            cdist.exec.run_or_fail(["/bin/true"])
-        except cdist.Error:
-            failed = True
-        else:
-            failed = False
-
-        self.assertFalse(failed)
-
-    def test_local_fail(self):
-        self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])
 
 class Config(unittest.TestCase):
     def setUp(self):
@@ -90,6 +41,7 @@ class Config(unittest.TestCase):
         self.config = cdist.config.Config("localhost",
                             initial_manifest=self.init_manifest,
                             exec_path=cdist_exec_path)
+        self.config.link_emulator()
 
     def test_initial_manifest_different_parameter(self):
         manifest_fd = open(self.init_manifest, "w")
@@ -121,6 +73,14 @@ class Config(unittest.TestCase):
 
         self.assertRaises(cdist.Error, self.config.run_initial_manifest)
 
+    def test_initial_manifest_non_existent_command(self):
+        manifest_fd = open(self.init_manifest, "w")
+        manifest_fd.writelines(["#!/bin/sh\n",
+            "thereisdefinitelynosuchcommend"])
+        manifest_fd.close()
+
+        self.assertRaises(cdist.Error, self.config.run_initial_manifest)
+
     def test_initial_manifest_parameter_twice(self):
         manifest_fd = open(self.init_manifest, "w")
         manifest_fd.writelines(["#!/bin/sh\n",
@@ -138,5 +98,4 @@ class Config(unittest.TestCase):
 
         self.assertFalse(failed)
 
-if __name__ == '__main__':
+
-    unittest.main()

test/test_exec.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+import os
+import sys
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+sys.path.insert(0, os.path.abspath(
+        os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
+
+import cdist.exec
+
+class Exec(unittest.TestCase):
+    def setUp(self):
+        """Create shell code and co."""
+
+        self.temp_dir = tempfile.mkdtemp()
+        self.shell_false = os.path.join(self.temp_dir, "shell_false")
+        self.shell_true  = os.path.join(self.temp_dir, "shell_true")
+
+        true_fd = open(self.shell_true, "w")
+        true_fd.writelines(["#!/bin/sh\n", "/bin/true"])
+        true_fd.close()
+        
+        false_fd = open(self.shell_false, "w")
+        false_fd.writelines(["#!/bin/sh\n", "/bin/false"])
+        false_fd.close()
+
+    def tearDown(self):
+        shutil.rmtree(self.temp_dir)
+        
+    def test_local_success_shell(self):
+        try:
+            cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true])
+        except cdist.Error:
+            failed = True
+        else:
+            failed = False
+
+        self.assertFalse(failed)
+
+    def test_local_fail_shell(self):
+        self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail,
+            self.shell_false, [self.shell_false])
+
+    def test_local_success(self):
+        try:
+            cdist.exec.run_or_fail(["/bin/true"])
+        except cdist.Error:
+            failed = True
+        else:
+            failed = False
+
+        self.assertFalse(failed)
+
+    def test_local_fail(self):
+        self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])