diff --git a/README b/README index daa12f70..3dff690f 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ "P' "" "" -[[!toc levels=2]] +[[!toc levels=3]] ## Introduction @@ -86,13 +86,46 @@ cdist was tested or is know to run on at least * SSH-Server -## Getting cdist +## Installation + +### Preperation + +Ensure you have Python 3.x and the **argparse** module installed on +the machine you use to **deploy to the targets**. + +#### Archlinux + +Archlinux already has python >= 3.2, so you only need to do: + + pacman -S python + +#### Debian + + aptitude install python3 python3-setuptools + easy_install3 argparse + + +#### Gentoo + +Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3). +If you want to ensure nothing breaks you must set back the python version to what was default before. + + emerge -av =python-3.2.2 --autounmask-write + emerge -av =python-3.2.2 + eselect python list + eselect python list set python3.2 + +#### Max OS X + +Ensure you have port installed and configured (http://www.macports.org/install.php). + + port install python32 + ln -s /opt/local/bin/python3.2 /opt/local/bin/python3 + +### Get cdist You can clone cdist from git, which gives you the advantage of having a version control in place for development of your own stuff as well. - -### Installation - To install cdist, execute the following commands: git clone git://git.schottelius.org/cdist diff --git a/bin/cdist b/bin/cdist index 245b2fc0..0bf3ed9c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -26,7 +26,7 @@ import os import re import sys -log = logging.getLogger(__name__) +log = logging.getLogger("cdist") # Ensure our /lib/ is included into PYTHON_PATH sys.path.insert(0, os.path.abspath( @@ -39,12 +39,17 @@ def commandline(): # Construct parser others can reuse parser = {} # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') + parser['loglevel'] = argparse.ArgumentParser(add_help=False) + parser['loglevel'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true', + default=False) + parser['loglevel'].add_argument('-v', '--verbose', + help='Set log level to info, be more verbose', + action='store_true', default=False) # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) + parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION, + parents=[parser['loglevel']]) parser['main'].add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + cdist.VERSION) @@ -52,7 +57,7 @@ def commandline(): # Banner parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) + parents=[parser['loglevel']]) parser['banner'].set_defaults(func=cdist.banner.banner) # Config and install (common stuff) @@ -74,12 +79,12 @@ def commandline(): # Config parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) + parents=[parser['loglevel'], parser['configinstall']]) parser['config'].set_defaults(func=cdist.config.config) # Install parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) + parents=[parser['loglevel'], parser['configinstall']]) parser['install'].set_defaults(func=cdist.install.install) for p in parser: @@ -87,17 +92,18 @@ def commandline(): args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) + # Loglevels are handled globally in here and debug wins over verbose + if args.verbose: + logging.root.setLevel(logging.INFO) + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) args.func(args) if __name__ == "__main__": try: - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + logging.basicConfig(format='%(levelname)s: %(message)s') if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): import cdist.emulator diff --git a/build.sh b/build.sh index d20e0211..021fb480 100755 --- a/build.sh +++ b/build.sh @@ -126,6 +126,14 @@ case "$1" in | xargs rm -f ;; + test) + python3 -m unittest discover test 'test_*.py' + ;; + + test-all) + python3 -m unittest discover test '*.py' + ;; + *) echo '' echo 'Welcome to cdist!' diff --git a/doc/changelog b/doc/changelog index a08efb34..9fb11307 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,10 @@ -2.0.2: +2.0.3: + * Improved logging, added --verbose, by more quiet by default + +2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) + * Add support for __debug variable in manifests + * Bugfix core: Various issues with type emulator 2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 8be1da54..4cdb6fcf 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -34,6 +34,8 @@ USER INTERFACE -> given after manifest run already! - use absent/present for state by default? +- buggy output with packages that don't exist in archlinux and fedora: + python3 vs. python TYPES ------ diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index b7748949..11c734f9 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -311,15 +311,13 @@ eof via __global/ - Support parallel execution - - and maximum number of parallel runs (-p X) - error handling / report failed hosts -- Allow manifest to be read from stdin - Create new video for cdist 2.0.0 http://www.youtube.com/watch?v=PRMjzy48eTI - Setup __debug, if -d is given, so other tools can reuse it - (-> non core feature! + - implement everywhere to external! - remote_prefix: scp vs. ssh issue diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index c205bdcc..7196c3b3 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -166,6 +166,11 @@ changed:: ENVIRONMENT VARIABLES --------------------- +__debug:: + If this variable is setup, cdist runs in debug mode. + You can use this information, to only output stuff in debug + mode as well. + Available for: initial manifest, type manifest __explorer:: Directory that contains all global explorers. Available for: explorer diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 192e5001..a0ca2ba2 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -19,7 +19,7 @@ # # -VERSION = "2.0.2" +VERSION = "2.0.3" class Error(Exception): """Base exception class for this project""" diff --git a/lib/cdist/config.py b/lib/cdist/config.py index aef0b28a..51615c28 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -65,6 +65,7 @@ class Config: def run_global_explores(self): """Run global explorers""" + log.info("Running global explorers") explorers = self.path.list_global_explorers() if(len(explorers) == 0): raise CdistError("No explorers found in", self.path.global_explorer_dir) @@ -107,18 +108,22 @@ class Config: cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + def init_deploy(self): """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) + self.link_emulator() def run_initial_manifest(self): """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) env = { "__manifest" : self.path.manifest_dir } self.run_manifest(self.path.initial_manifest, extra_env=env) @@ -146,6 +151,10 @@ class Config: env['__target_host'] = self.target_host env['__global'] = self.path.out_dir + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" + # Required for recording source env['__cdist_manifest'] = manifest @@ -235,12 +244,13 @@ class Config: self.run_global_explores() self.run_initial_manifest() + log.info("Running object manifests and type explorers") + old_objects = [] objects = self.path.list_objects() # Continue process until no new objects are created anymore while old_objects != objects: - log.debug("Prepare stage") old_objects = list(objects) for cdist_object in objects: if cdist_object in self.objects_prepared: @@ -255,7 +265,7 @@ class Config: def stage_run(self): """The final (and real) step of deployment""" - log.debug("Actual run objects") + log.info("Generating and executing code") # Now do the final steps over the existing objects for cdist_object in self.path.list_objects(): log.debug("Run object: %s", cdist_object) @@ -298,7 +308,7 @@ def config(args): if args.parallel: for p in process.keys(): - log.debug("Joining %s", p) + log.debug("Joining process %s", p) process[p].join() time_end = datetime.datetime.now() diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 68a67176..38a58f8c 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -36,20 +36,21 @@ def run(argv): global_dir = os.environ['__global'] object_source = os.environ['__cdist_manifest'] + if '__debug' in os.environ: + logging.root.setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + parser = argparse.ArgumentParser(add_help=False) - # Setup optional parameters for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): argument = "--" + parameter parser.add_argument(argument, action='store', required=False) - - # Setup required parameters for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): argument = "--" + parameter parser.add_argument(argument, action='store', required=True) - # Setup positional parameter, if not singleton - + # If not singleton support one positional parameter if not os.path.isfile(os.path.join(type_dir, "singleton")): parser.add_argument("object_id", nargs=1) @@ -67,6 +68,10 @@ def run(argv): if object_id[0] == '/': object_id = object_id[1:] + # Prefix output by object_self + logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s' + logging.basicConfig(format=logformat) + # FIXME: verify object id log.debug(args) @@ -110,12 +115,12 @@ def run(argv): value_old = param_fd.readlines() param_fd.close() - if(value_old != value): - raise cdist.Error("Parameter + \"" + param + + if(value_old[0] != value): + raise cdist.Error("Parameter\"" + param + "\" differs: " + " ".join(value_old) + " vs. " + value + "\nSource = " + " ".join(old_object_source) - + " new =" + object_source) + + " new = " + object_source) else: param_fd = open(file, "w") param_fd.writelines(value) @@ -124,7 +129,7 @@ def run(argv): # Record requirements if "__require" in os.environ: requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) + log.debug(object_id + ":Writing requirements: " + requirements) require_fd = open(os.path.join(object_dir, "require"), "a") require_fd.writelines(requirements.split(" ")) require_fd.close() @@ -134,7 +139,7 @@ def run(argv): source_fd.writelines(object_source) source_fd.close() - print("Finished " + type + "/" + object_id + repr(params)) + log.debug("Finished " + type + "/" + object_id + repr(params)) def link(exec_path, bin_dir, type_list): diff --git a/test/nico_ui.py b/test/nico_ui.py new file mode 100755 index 00000000..8ce98043 --- /dev/null +++ b/test/nico_ui.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import subprocess +import unittest + +cdist_commands=["banner", "config", "install"] + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "../bin/cdist")) + +class UI(unittest.TestCase): + def test_banner(self): + self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + + def test_help(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + + # FIXME: mockup needed + def test_config_localhost(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + diff --git a/test.py b/test/test_config.py old mode 100755 new mode 100644 similarity index 67% rename from test.py rename to test/test_config.py index 8a797d98..0632ebcc --- a/test.py +++ b/test/test_config.py @@ -20,68 +20,19 @@ # # - import os import sys -import shutil import tempfile import unittest sys.path.insert(0, os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))) + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) + +import cdist.config cdist_exec_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) -import cdist -import cdist.config -import cdist.exec - -class Exec(unittest.TestCase): - def setUp(self): - """Create shell code and co.""" - - self.temp_dir = tempfile.mkdtemp() - self.shell_false = os.path.join(self.temp_dir, "shell_false") - self.shell_true = os.path.join(self.temp_dir, "shell_true") - - true_fd = open(self.shell_true, "w") - true_fd.writelines(["#!/bin/sh\n", "/bin/true"]) - true_fd.close() - - false_fd = open(self.shell_false, "w") - false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) - false_fd.close() - - def tearDown(self): - shutil.rmtree(self.temp_dir) - - def test_local_success_shell(self): - try: - cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) - except cdist.Error: - failed = True - else: - failed = False - - self.assertFalse(failed) - - def test_local_fail_shell(self): - self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, - self.shell_false, [self.shell_false]) - - def test_local_success(self): - try: - cdist.exec.run_or_fail(["/bin/true"]) - except cdist.Error: - failed = True - else: - failed = False - - self.assertFalse(failed) - - def test_local_fail(self): - self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) class Config(unittest.TestCase): def setUp(self): @@ -90,6 +41,7 @@ class Config(unittest.TestCase): self.config = cdist.config.Config("localhost", initial_manifest=self.init_manifest, exec_path=cdist_exec_path) + self.config.link_emulator() def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") @@ -121,6 +73,14 @@ class Config(unittest.TestCase): self.assertRaises(cdist.Error, self.config.run_initial_manifest) + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", @@ -138,5 +98,4 @@ class Config(unittest.TestCase): self.assertFalse(failed) -if __name__ == '__main__': - unittest.main() + diff --git a/test/test_exec.py b/test/test_exec.py new file mode 100755 index 00000000..901b5efd --- /dev/null +++ b/test/test_exec.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import sys +import shutil +import subprocess +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) + +import cdist.exec + +class Exec(unittest.TestCase): + def setUp(self): + """Create shell code and co.""" + + self.temp_dir = tempfile.mkdtemp() + self.shell_false = os.path.join(self.temp_dir, "shell_false") + self.shell_true = os.path.join(self.temp_dir, "shell_true") + + true_fd = open(self.shell_true, "w") + true_fd.writelines(["#!/bin/sh\n", "/bin/true"]) + true_fd.close() + + false_fd = open(self.shell_false, "w") + false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) + false_fd.close() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_local_success_shell(self): + try: + cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail_shell(self): + self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, + self.shell_false, [self.shell_false]) + + def test_local_success(self): + try: + cdist.exec.run_or_fail(["/bin/true"]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail(self): + self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"])