From 1d981200c8cfaff9da4a301a261525649bab2da7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 15:56:43 +0200 Subject: [PATCH 01/29] test breaking run, if a non existent command is called Signed-off-by: Nico Schottelius --- test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test.py b/test.py index 8a797d98..1e9aee48 100755 --- a/test.py +++ b/test.py @@ -121,6 +121,14 @@ class Config(unittest.TestCase): self.assertRaises(cdist.Error, self.config.run_initial_manifest) + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", From ccbd0f1d8450af180f5883f3b4dad0d429dc35d6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:05:26 +0200 Subject: [PATCH 02/29] introduce config.link_emulator() to be called from test Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 11 +++++++---- test.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index aef0b28a..05bdb2a1 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -107,15 +107,18 @@ class Config: cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + def init_deploy(self): """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) + self.link_emulator() def run_initial_manifest(self): """Run the initial manifest""" @@ -149,7 +152,7 @@ class Config: # Required for recording source env['__cdist_manifest'] = manifest - # Required to find types + # Required to find types env['__cdist_type_base_dir'] = self.path.type_base_dir # Other environment stuff diff --git a/test.py b/test.py index 1e9aee48..ce433152 100755 --- a/test.py +++ b/test.py @@ -90,6 +90,7 @@ class Config(unittest.TestCase): self.config = cdist.config.Config("localhost", initial_manifest=self.init_manifest, exec_path=cdist_exec_path) + self.config.link_emulator() def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") From 272b8722c1cdda9fa7f1ef349abe9cdc9366945d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:10:53 +0200 Subject: [PATCH 03/29] fix output, compare string with string not string with list Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 6 +++--- test.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 68a67176..6a99ba79 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -110,12 +110,12 @@ def run(argv): value_old = param_fd.readlines() param_fd.close() - if(value_old != value): - raise cdist.Error("Parameter + \"" + param + + if(value_old[0] != value): + raise cdist.Error("Parameter\"" + param + "\" differs: " + " ".join(value_old) + " vs. " + value + "\nSource = " + " ".join(old_object_source) - + " new =" + object_source) + + " new = " + object_source) else: param_fd = open(file, "w") param_fd.writelines(value) diff --git a/test.py b/test.py index ce433152..b6a575e9 100755 --- a/test.py +++ b/test.py @@ -139,7 +139,9 @@ class Config(unittest.TestCase): manifest_fd.close() try: + print("a") self.config.run_initial_manifest() + print("b") except cdist.Error: failed = True else: From a1655856999a06fae352e61bf54d6b29561d58dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:20:46 +0200 Subject: [PATCH 04/29] pass __debug to manifest Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 5 ++++- lib/cdist/emulator.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 05bdb2a1..6df5d6d8 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -149,10 +149,13 @@ class Config: env['__target_host'] = self.target_host env['__global'] = self.path.out_dir + # Submit debug flag to manifest, can be used by emulator and types + env['__debug'] = "yes" + # Required for recording source env['__cdist_manifest'] = manifest - # Required to find types + # Required to find types env['__cdist_type_base_dir'] = self.path.type_base_dir # Other environment stuff diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 6a99ba79..9fc7c576 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -36,6 +36,9 @@ def run(argv): global_dir = os.environ['__global'] object_source = os.environ['__cdist_manifest'] + if '__debug' in os.environ: + logging.root.setLevel(logging.DEBUG) + parser = argparse.ArgumentParser(add_help=False) # Setup optional parameters @@ -134,7 +137,7 @@ def run(argv): source_fd.writelines(object_source) source_fd.close() - print("Finished " + type + "/" + object_id + repr(params)) + log.debug("Finished " + type + "/" + object_id + repr(params)) def link(exec_path, bin_dir, type_list): From b63fdf16edb71129908fedda0fd82e06d33d7b7a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:21:36 +0200 Subject: [PATCH 05/29] document __debug in reference Signed-off-by: Nico Schottelius --- doc/man/cdist-reference.text.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index c205bdcc..7196c3b3 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -166,6 +166,11 @@ changed:: ENVIRONMENT VARIABLES --------------------- +__debug:: + If this variable is setup, cdist runs in debug mode. + You can use this information, to only output stuff in debug + mode as well. + Available for: initial manifest, type manifest __explorer:: Directory that contains all global explorers. Available for: explorer From ae7887f775b8db8b1f89f1798de22dc15f9aa98a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:25:58 +0200 Subject: [PATCH 06/29] ++changes Signed-off-by: Nico Schottelius --- doc/changelog | 2 ++ lib/cdist/emulator.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index a08efb34..53ea04a4 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,7 @@ 2.0.2: * Add support for detection of OpenWall Linux (Matthias Teege) + * Add support for __debug variable in manifests + * Bugfix core: Various issues with type emulator 2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 9fc7c576..d69694df 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -127,7 +127,7 @@ def run(argv): # Record requirements if "__require" in os.environ: requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) + log.debug(object_id + ":Writing requirements: " + requirements) require_fd = open(os.path.join(object_dir, "require"), "a") require_fd.writelines(requirements.split(" ")) require_fd.close() From b52939ccfedf2b4e536f103cd883f61b19cf67a9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:40:50 +0200 Subject: [PATCH 07/29] only debug if __debug is setup Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 3 ++- lib/cdist/emulator.py | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 6df5d6d8..af47d091 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -150,7 +150,8 @@ class Config: env['__global'] = self.path.out_dir # Submit debug flag to manifest, can be used by emulator and types - env['__debug'] = "yes" + if self.debug: + env['__debug'] = "yes" # Required for recording source env['__cdist_manifest'] = manifest diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index d69694df..38a58f8c 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -38,21 +38,19 @@ def run(argv): if '__debug' in os.environ: logging.root.setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(add_help=False) - # Setup optional parameters for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): argument = "--" + parameter parser.add_argument(argument, action='store', required=False) - - # Setup required parameters for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): argument = "--" + parameter parser.add_argument(argument, action='store', required=True) - # Setup positional parameter, if not singleton - + # If not singleton support one positional parameter if not os.path.isfile(os.path.join(type_dir, "singleton")): parser.add_argument("object_id", nargs=1) @@ -70,6 +68,10 @@ def run(argv): if object_id[0] == '/': object_id = object_id[1:] + # Prefix output by object_self + logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s' + logging.basicConfig(format=logformat) + # FIXME: verify object id log.debug(args) From 67f45944dd7d99b4a7710f34dc419ce760d65047 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:43:53 +0200 Subject: [PATCH 08/29] less todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index b7748949..11c734f9 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -311,15 +311,13 @@ eof via __global/ - Support parallel execution - - and maximum number of parallel runs (-p X) - error handling / report failed hosts -- Allow manifest to be read from stdin - Create new video for cdist 2.0.0 http://www.youtube.com/watch?v=PRMjzy48eTI - Setup __debug, if -d is given, so other tools can reuse it - (-> non core feature! + - implement everywhere to external! - remote_prefix: scp vs. ssh issue From 4a0e0ceb8a9ec695ae5e0c7c0760dc5684d82d0f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:44:38 +0200 Subject: [PATCH 09/29] release 2.0.2 today Signed-off-by: Nico Schottelius --- doc/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index 53ea04a4..39455351 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,4 +1,4 @@ -2.0.2: +2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) * Add support for __debug variable in manifests * Bugfix core: Various issues with type emulator From 6519d7e92c72ff66ee2fcaeeaa62e2fb45424269 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:53:02 +0200 Subject: [PATCH 10/29] add hint for debian/argparse Signed-off-by: Nico Schottelius --- README | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README b/README index daa12f70..7c089cf3 100644 --- a/README +++ b/README @@ -86,13 +86,23 @@ cdist was tested or is know to run on at least * SSH-Server -## Getting cdist +## Installation + +### Preperation + +Ensure you have Python 3.x and the **argparse** module installed on +the machine you use to **deploy to the targets**. + +#### Debian + + aptitude install python3 python3-setuptools + easy_install3 argparse + + +### Get cdist You can clone cdist from git, which gives you the advantage of having a version control in place for development of your own stuff as well. - -### Installation - To install cdist, execute the following commands: git clone git://git.schottelius.org/cdist From dfc3d4c7c87bc6ce0e601940a6e4417f037684da Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:55:39 +0200 Subject: [PATCH 11/29] toc = level 3 (we are getting bigger) Signed-off-by: Nico Schottelius --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 7c089cf3..555d28d6 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ "P' "" "" -[[!toc levels=2]] +[[!toc levels=3]] ## Introduction From 6c9bf9e5d7d527f4e098272e68e5ad67d6de0322 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:12:08 +0200 Subject: [PATCH 12/29] use cdist, not __main__ as logger name Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 245b2fc0..c79db293 100755 --- a/bin/cdist +++ b/bin/cdist @@ -26,7 +26,7 @@ import os import re import sys -log = logging.getLogger(__name__) +log = logging.getLogger("cdist") # Ensure our /lib/ is included into PYTHON_PATH sys.path.insert(0, os.path.abspath( From d8da7635def25dc9873886681345d67418e80de5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:15:54 +0200 Subject: [PATCH 13/29] use warning loglevel by default Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- lib/cdist/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index c79db293..ea7cf4f9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -97,7 +97,7 @@ def commandline(): if __name__ == "__main__": try: - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + logging.basicConfig(format='%(levelname)s: %(message)s') if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): import cdist.emulator diff --git a/lib/cdist/config.py b/lib/cdist/config.py index af47d091..f50195aa 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -65,6 +65,7 @@ class Config: def run_global_explores(self): """Run global explorers""" + log.info("Running global explorers") explorers = self.path.list_global_explorers() if(len(explorers) == 0): raise CdistError("No explorers found in", self.path.global_explorer_dir) From 0e8dcb2f3db257e5e76f89b4b8394e3ceaa09a4b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:20:31 +0200 Subject: [PATCH 14/29] add verbose support Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index ea7cf4f9..7d78ff51 100755 --- a/bin/cdist +++ b/bin/cdist @@ -41,7 +41,11 @@ def commandline(): # Options _all_ parsers have in common parser['most'] = argparse.ArgumentParser(add_help=False) parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') + help='Set log level to debug', action='store_true', + default=False) + parser['most'].add_argument('-v', '--verbose', + help='Set log level to info, is more verbose', + action='store_true', default=False) # Main subcommand parser parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) @@ -87,12 +91,13 @@ def commandline(): args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) + # Loglevels are handled globally in here and debug wins over verbose + if args.verbose: + logging.root.setLevel(logging.INFO) + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) args.func(args) if __name__ == "__main__": From d15d65924835d0c0f1c4d559bf44fcc9cdccb21c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:20:43 +0200 Subject: [PATCH 15/29] increment version for next release Signed-off-by: Nico Schottelius --- lib/cdist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 192e5001..a0ca2ba2 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -19,7 +19,7 @@ # # -VERSION = "2.0.2" +VERSION = "2.0.3" class Error(Exception): """Base exception class for this project""" From 26c0d5d9aa28b67b1da47e326bdc3facc045d29a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:27:28 +0200 Subject: [PATCH 16/29] upcoming changes for 2.0.3 Signed-off-by: Nico Schottelius --- doc/changelog | 3 +++ lib/cdist/config.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/changelog b/doc/changelog index 39455351..9fb11307 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,3 +1,6 @@ +2.0.3: + * Improved logging, added --verbose, by more quiet by default + 2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) * Add support for __debug variable in manifests diff --git a/lib/cdist/config.py b/lib/cdist/config.py index f50195aa..51615c28 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -123,6 +123,7 @@ class Config: def run_initial_manifest(self): """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) env = { "__manifest" : self.path.manifest_dir } self.run_manifest(self.path.initial_manifest, extra_env=env) @@ -243,12 +244,13 @@ class Config: self.run_global_explores() self.run_initial_manifest() + log.info("Running object manifests and type explorers") + old_objects = [] objects = self.path.list_objects() # Continue process until no new objects are created anymore while old_objects != objects: - log.debug("Prepare stage") old_objects = list(objects) for cdist_object in objects: if cdist_object in self.objects_prepared: @@ -263,7 +265,7 @@ class Config: def stage_run(self): """The final (and real) step of deployment""" - log.debug("Actual run objects") + log.info("Generating and executing code") # Now do the final steps over the existing objects for cdist_object in self.path.list_objects(): log.debug("Run object: %s", cdist_object) @@ -306,7 +308,7 @@ def config(args): if args.parallel: for p in process.keys(): - log.debug("Joining %s", p) + log.debug("Joining process %s", p) process[p].join() time_end = datetime.datetime.now() From 298688eb35eef7b4faee8f538062145272b7308d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:31:39 +0200 Subject: [PATCH 17/29] hint for archlinux users Signed-off-by: Nico Schottelius --- README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README b/README index 555d28d6..28580c38 100644 --- a/README +++ b/README @@ -93,6 +93,12 @@ cdist was tested or is know to run on at least Ensure you have Python 3.x and the **argparse** module installed on the machine you use to **deploy to the targets**. +#### Archlinux + +Archlinux already has python >= 3.2, so you only need to do: + + pacman -S python + #### Debian aptitude install python3 python3-setuptools From fdb694bc42a09f94a71765c85d3fcaba74f2c70e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:36:01 +0200 Subject: [PATCH 18/29] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 8be1da54..4cdb6fcf 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -34,6 +34,8 @@ USER INTERFACE -> given after manifest run already! - use absent/present for state by default? +- buggy output with packages that don't exist in archlinux and fedora: + python3 vs. python TYPES ------ From e162861b17e3bd4287ddd55e8532bb00c49ffae1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:06:20 +0200 Subject: [PATCH 19/29] consistently support -h, -v, -d in all commands Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7d78ff51..0bf3ed9c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -39,16 +39,17 @@ def commandline(): # Construct parser others can reuse parser = {} # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', + parser['loglevel'] = argparse.ArgumentParser(add_help=False) + parser['loglevel'].add_argument('-d', '--debug', help='Set log level to debug', action='store_true', default=False) - parser['most'].add_argument('-v', '--verbose', - help='Set log level to info, is more verbose', + parser['loglevel'].add_argument('-v', '--verbose', + help='Set log level to info, be more verbose', action='store_true', default=False) # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) + parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION, + parents=[parser['loglevel']]) parser['main'].add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + cdist.VERSION) @@ -56,7 +57,7 @@ def commandline(): # Banner parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) + parents=[parser['loglevel']]) parser['banner'].set_defaults(func=cdist.banner.banner) # Config and install (common stuff) @@ -78,12 +79,12 @@ def commandline(): # Config parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) + parents=[parser['loglevel'], parser['configinstall']]) parser['config'].set_defaults(func=cdist.config.config) # Install parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) + parents=[parser['loglevel'], parser['configinstall']]) parser['install'].set_defaults(func=cdist.install.install) for p in parser: From 5d018393a9a25e41121d12c8697a0c7368cd1c2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:13:39 +0200 Subject: [PATCH 20/29] test if banner works Signed-off-by: Nico Schottelius --- test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test.py b/test.py index b6a575e9..8fe759a6 100755 --- a/test.py +++ b/test.py @@ -24,6 +24,7 @@ import os import sys import shutil +import subprocess import tempfile import unittest @@ -149,5 +150,11 @@ class Config(unittest.TestCase): self.assertFalse(failed) + +class UI(unittest.TestCase): + def test_banner(self): + self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + + if __name__ == '__main__': unittest.main() From 3265755bb9197e54e66de791d24b7eff70c980e1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:15:04 +0200 Subject: [PATCH 21/29] test help function Signed-off-by: Nico Schottelius --- test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test.py b/test.py index 8fe759a6..d03ad0f1 100755 --- a/test.py +++ b/test.py @@ -34,6 +34,8 @@ sys.path.insert(0, os.path.abspath( cdist_exec_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) +cdist_commands=["banner", "config", "install"] + import cdist import cdist.config import cdist.exec @@ -155,6 +157,10 @@ class UI(unittest.TestCase): def test_banner(self): self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + def test_help(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + if __name__ == '__main__': unittest.main() From f357eb3038c9d5eef63c506384011f1d0a7e0067 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:17:27 +0200 Subject: [PATCH 22/29] test config of localhost Signed-off-by: Nico Schottelius --- test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test.py b/test.py index d03ad0f1..5d57fae4 100755 --- a/test.py +++ b/test.py @@ -161,6 +161,11 @@ class UI(unittest.TestCase): for cmd in cdist_commands: self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + # FIXME: mockup needed + def test_config_localhost(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + if __name__ == '__main__': unittest.main() From ab061770f4c4fa2b5fdde1fea46a73c156776278 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Sat, 1 Oct 2011 13:08:07 +0200 Subject: [PATCH 23/29] Installation with gentoo --- README | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README b/README index 28580c38..aab4c3a5 100644 --- a/README +++ b/README @@ -105,6 +105,16 @@ Archlinux already has python >= 3.2, so you only need to do: easy_install3 argparse +#### Gentoo + +Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3). +If you want to ensure nothing breaks you must set back the python version to what was default before. + + emerge -av =python-3.2.2 --autounmask-write + emerge -av =python-3.2.2 + eselect python list + eselect python list set python3.2 + ### Get cdist You can clone cdist from git, which gives you the advantage of having From 7d290a7755532a66618caa94093db1a040d90492 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Oct 2011 16:34:25 +0200 Subject: [PATCH 24/29] begin to splitup test cases Signed-off-by: Nico Schottelius --- test.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index 5d57fae4..1e0973c5 100755 --- a/test.py +++ b/test.py @@ -142,9 +142,7 @@ class Config(unittest.TestCase): manifest_fd.close() try: - print("a") self.config.run_initial_manifest() - print("b") except cdist.Error: failed = True else: @@ -166,6 +164,30 @@ class UI(unittest.TestCase): for cmd in cdist_commands: self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) - + +def almost_all_tests(): + suite = unittest.TestSuite([ + unittest.TestLoader().loadTestsFromTestCase(Config), + unittest.TestLoader().loadTestsFromTestCase(Exec)]) + + return suite + +def all_tests(): + suite = unittest.defaultTestLoader + return suite + if __name__ == '__main__': - unittest.main() + result = unittest.TestResult() + # only run some tests, when giving -a -> stuff that usually breaks + if len(sys.argv) >= 2: + if sys.argv[1] == "-a": + suite = all_tests(); + else: + sys.exit(1) + else: + suite = almost_all_tests(); + + # suite.run(result) + # unittest.main() + # unittest.TextTestRunner().run(suite) + From f3bedd8d93b56fc3c2c3aa29b6638ed441f972d9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Oct 2011 17:25:12 +0200 Subject: [PATCH 25/29] move tests to test/ again, but seperate them now and add ./build.sh test Signed-off-by: Nico Schottelius --- build.sh | 4 ++ test/test_config.py | 101 ++++++++++++++++++++++++++++++++++++++++++++ test/test_exec.py | 80 +++++++++++++++++++++++++++++++++++ 3 files changed, 185 insertions(+) create mode 100644 test/test_config.py create mode 100755 test/test_exec.py diff --git a/build.sh b/build.sh index d20e0211..24c907e6 100755 --- a/build.sh +++ b/build.sh @@ -126,6 +126,10 @@ case "$1" in | xargs rm -f ;; + test) + python3 -m unittest discover test 'test_*.py' + ;; + *) echo '' echo 'Welcome to cdist!' diff --git a/test/test_config.py b/test/test_config.py new file mode 100644 index 00000000..0632ebcc --- /dev/null +++ b/test/test_config.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) + +import cdist.config + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + + +class Config(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.config = cdist.config.Config("localhost", + initial_manifest=self.init_manifest, + exec_path=cdist_exec_path) + self.config.link_emulator() + + def test_initial_manifest_different_parameter(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_added(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + '\n', + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_removed(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + "\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_twice(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + try: + self.config.run_initial_manifest() + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + diff --git a/test/test_exec.py b/test/test_exec.py new file mode 100755 index 00000000..901b5efd --- /dev/null +++ b/test/test_exec.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import sys +import shutil +import subprocess +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) + +import cdist.exec + +class Exec(unittest.TestCase): + def setUp(self): + """Create shell code and co.""" + + self.temp_dir = tempfile.mkdtemp() + self.shell_false = os.path.join(self.temp_dir, "shell_false") + self.shell_true = os.path.join(self.temp_dir, "shell_true") + + true_fd = open(self.shell_true, "w") + true_fd.writelines(["#!/bin/sh\n", "/bin/true"]) + true_fd.close() + + false_fd = open(self.shell_false, "w") + false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) + false_fd.close() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_local_success_shell(self): + try: + cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail_shell(self): + self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, + self.shell_false, [self.shell_false]) + + def test_local_success(self): + try: + cdist.exec.run_or_fail(["/bin/true"]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail(self): + self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) From 390342b324c0b7bca68bdcf381c37bac98c5c326 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Sat, 1 Oct 2011 14:02:45 +0200 Subject: [PATCH 26/29] Mac OS X Tutorial --- README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README b/README index aab4c3a5..2c6465f2 100644 --- a/README +++ b/README @@ -115,6 +115,12 @@ If you want to ensure nothing breaks you must set back the python version to wha eselect python list eselect python list set python3.2 +#### Max OS X + +Ensure you have port installed and configured (http://www.macports.org/install.php). + + port install python32 + ### Get cdist You can clone cdist from git, which gives you the advantage of having From 6425f1d55c86d383464d64e177eb2d28ac544f41 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Sat, 1 Oct 2011 14:46:41 +0200 Subject: [PATCH 27/29] Finished Mac OS X installation --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index 2c6465f2..3dff690f 100644 --- a/README +++ b/README @@ -120,6 +120,7 @@ If you want to ensure nothing breaks you must set back the python version to wha Ensure you have port installed and configured (http://www.macports.org/install.php). port install python32 + ln -s /opt/local/bin/python3.2 /opt/local/bin/python3 ### Get cdist From df727a6b6bf52bd35c820480640ec651c3ede841 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 2 Oct 2011 15:52:19 +0200 Subject: [PATCH 28/29] remove everything but ui from ui test Signed-off-by: Nico Schottelius --- test.py | 193 ------------------------------------------------ test/nico_ui.py | 41 ++++++++++ 2 files changed, 41 insertions(+), 193 deletions(-) delete mode 100755 test.py create mode 100755 test/nico_ui.py diff --git a/test.py b/test.py deleted file mode 100755 index 1e0973c5..00000000 --- a/test.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - - -import os -import sys -import shutil -import subprocess -import tempfile -import unittest - -sys.path.insert(0, os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))) - -cdist_exec_path = os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) - -cdist_commands=["banner", "config", "install"] - -import cdist -import cdist.config -import cdist.exec - -class Exec(unittest.TestCase): - def setUp(self): - """Create shell code and co.""" - - self.temp_dir = tempfile.mkdtemp() - self.shell_false = os.path.join(self.temp_dir, "shell_false") - self.shell_true = os.path.join(self.temp_dir, "shell_true") - - true_fd = open(self.shell_true, "w") - true_fd.writelines(["#!/bin/sh\n", "/bin/true"]) - true_fd.close() - - false_fd = open(self.shell_false, "w") - false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) - false_fd.close() - - def tearDown(self): - shutil.rmtree(self.temp_dir) - - def test_local_success_shell(self): - try: - cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) - except cdist.Error: - failed = True - else: - failed = False - - self.assertFalse(failed) - - def test_local_fail_shell(self): - self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, - self.shell_false, [self.shell_false]) - - def test_local_success(self): - try: - cdist.exec.run_or_fail(["/bin/true"]) - except cdist.Error: - failed = True - else: - failed = False - - self.assertFalse(failed) - - def test_local_fail(self): - self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) - -class Config(unittest.TestCase): - def setUp(self): - self.temp_dir = tempfile.mkdtemp() - self.init_manifest = os.path.join(self.temp_dir, "manifest") - self.config = cdist.config.Config("localhost", - initial_manifest=self.init_manifest, - exec_path=cdist_exec_path) - self.config.link_emulator() - - def test_initial_manifest_different_parameter(self): - manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + " --mode 0700\n", - "__file " + self.temp_dir + " --mode 0600\n", - ]) - manifest_fd.close() - - self.assertRaises(cdist.Error, self.config.run_initial_manifest) - - def test_initial_manifest_parameter_added(self): - manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + '\n', - "__file " + self.temp_dir + " --mode 0600\n", - ]) - manifest_fd.close() - - self.assertRaises(cdist.Error, self.config.run_initial_manifest) - - def test_initial_manifest_parameter_removed(self): - manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + " --mode 0600\n", - "__file " + self.temp_dir + "\n", - ]) - manifest_fd.close() - - self.assertRaises(cdist.Error, self.config.run_initial_manifest) - - def test_initial_manifest_non_existent_command(self): - manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh\n", - "thereisdefinitelynosuchcommend"]) - manifest_fd.close() - - self.assertRaises(cdist.Error, self.config.run_initial_manifest) - - def test_initial_manifest_parameter_twice(self): - manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + " --mode 0600\n", - "__file " + self.temp_dir + " --mode 0600\n", - ]) - manifest_fd.close() - - try: - self.config.run_initial_manifest() - except cdist.Error: - failed = True - else: - failed = False - - self.assertFalse(failed) - - -class UI(unittest.TestCase): - def test_banner(self): - self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) - - def test_help(self): - for cmd in cdist_commands: - self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) - - # FIXME: mockup needed - def test_config_localhost(self): - for cmd in cdist_commands: - self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) - - -def almost_all_tests(): - suite = unittest.TestSuite([ - unittest.TestLoader().loadTestsFromTestCase(Config), - unittest.TestLoader().loadTestsFromTestCase(Exec)]) - - return suite - -def all_tests(): - suite = unittest.defaultTestLoader - return suite - -if __name__ == '__main__': - result = unittest.TestResult() - # only run some tests, when giving -a -> stuff that usually breaks - if len(sys.argv) >= 2: - if sys.argv[1] == "-a": - suite = all_tests(); - else: - sys.exit(1) - else: - suite = almost_all_tests(); - - # suite.run(result) - # unittest.main() - # unittest.TextTestRunner().run(suite) - diff --git a/test/nico_ui.py b/test/nico_ui.py new file mode 100755 index 00000000..4e5b1a17 --- /dev/null +++ b/test/nico_ui.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import subprocess + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + +class UI(unittest.TestCase): + def test_banner(self): + self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + + def test_help(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + + # FIXME: mockup needed + def test_config_localhost(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + From cadb4fa852870f872fc65ea7e85680406eff20a5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 2 Oct 2011 15:56:27 +0200 Subject: [PATCH 29/29] add test all and make ui tests work again Signed-off-by: Nico Schottelius --- build.sh | 4 ++++ test/nico_ui.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 24c907e6..021fb480 100755 --- a/build.sh +++ b/build.sh @@ -130,6 +130,10 @@ case "$1" in python3 -m unittest discover test 'test_*.py' ;; + test-all) + python3 -m unittest discover test '*.py' + ;; + *) echo '' echo 'Welcome to cdist!' diff --git a/test/nico_ui.py b/test/nico_ui.py index 4e5b1a17..8ce98043 100755 --- a/test/nico_ui.py +++ b/test/nico_ui.py @@ -21,10 +21,14 @@ # +import os import subprocess +import unittest + +cdist_commands=["banner", "config", "install"] cdist_exec_path = os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + os.path.join(os.path.dirname(os.path.realpath(__file__)), "../bin/cdist")) class UI(unittest.TestCase): def test_banner(self):