From 5443db97c4d7199c04dfac5d15d37345d6caa7f1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 12:54:55 +0200 Subject: [PATCH 1/6] always setup __debug if debug is set Signed-off-by: Nico Schottelius --- doc/dev/todo/tests | 44 ++++++++++++++++++++++++++++++++----- lib/cdist/config_install.py | 28 ++++++++++------------- 2 files changed, 49 insertions(+), 23 deletions(-) diff --git a/doc/dev/todo/tests b/doc/dev/todo/tests index 6fcd1a04..27bca3b6 100644 --- a/doc/dev/todo/tests +++ b/doc/dev/todo/tests @@ -1,11 +1,43 @@ -Tests needed: +Tests needed for config_install: - - Fail if cache_dir from previous run cannot be deleted - - Fail if cache_dir cannot be created from current out_dir - - transfer_type_explorers: Explorers are not transferred twice - - transfer_type_explorers: No transfer tried if there are no type_explorers + __init__(): + - sets up env: __target_host + + cleanup() + - Fail if cache_dir from previous run cannot be deleted + - Fail if cache_dir cannot be created from current out_dir + + filter() + - ensure logformat is changed: target host is prefixed: + LOGLEVEL: target_host: MESSAGE + + run_initial_manifest(): + - parameter is actually used (from __init__) + - ensure changing the manifest actually runs a different manifest + -> give ConfigInstall Constructor different manifest + -> different manifest is executed. + - test all submitted (from core to type manifest) variables: + - ENVIRONMENT + - they are set + - they contain the correct values + + run_type_manifest(): + - test all submitted (from core to type manifest) variables: + - ENVIRONMENT + - they are set + - they contain the correct values + run_manifest(): + - test all submitted variables: + - ENVIRONMENT + - including __debug, if debug + - they are set + - they contain the correct values + - does $require work? + + transfer_type_explorers(): + - Explorers are not transferred twice + - No transfer tried if there are no type_explorers - - does $require work? - $whatever should fail if there is no global explorer directory - emulator may only be called with __ as prefix - fail otherwise! diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 2f1adecd..a286472b 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -49,20 +49,16 @@ class ConfigInstall: self.debug = debug - # Required for testing + # Only required for testing self.exec_path = exec_path # Configure logging log.addFilter(self) # Base and Temp Base - if base_path: - self.base_path = base_path - else: - self.base_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), - os.pardir, - os.pardir)) + self.base_path = (base_path or + self.base_path = os.path.abspath(os.path.join( + os.path.dirname(__file__), os.pardir, os.pardir)) # Local input self.cache_path = os.path.join(self.base_path, "cache", @@ -74,10 +70,8 @@ class ConfigInstall: self.type_base_path = os.path.join(self.conf_path, "type") self.lib_path = os.path.join(self.base_path, "lib") - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_path, "init") + self.initial_manifest = (initial_manifest or + os.path.join(self.manifest_path, "init")) # Local output if '__cdist_out_dir' in os.environ: @@ -108,7 +102,6 @@ class ConfigInstall: self.__init_remote_paths() - def __init_remote_paths(self): """Initialise remote directory structure""" self.remove_remote_path(self.remote_base_path) @@ -122,6 +115,7 @@ class ConfigInstall: if not os.path.isdir(self.base_path): os.mkdir(self.base_path) + # FIXME: raise more beautiful exception / Steven: handle exception os.mkdir(self.out_path) os.mkdir(self.global_explorer_out_path) os.mkdir(self.bin_path) @@ -129,6 +123,9 @@ class ConfigInstall: def __init_env(self): """Environment usable for other stuff""" os.environ['__target_host'] = self.target_host + if self.debug: + os.environ['__debug'] = "yes" + def cleanup(self): # Do not use in __del__: @@ -137,6 +134,7 @@ class ConfigInstall: # or in the process of being torn down (e.g. the import machinery shutting down)" # log.debug("Saving " + self.out_path + " to " + self.cache_path) + # FIXME: raise more beautiful exception / Steven: handle exception # Remove previous cache if os.path.exists(self.cache_path): shutil.rmtree(self.cache_path) @@ -182,10 +180,6 @@ class ConfigInstall: env['__target_host'] = self.target_host env['__global'] = self.out_path - # Submit debug flag to manifest, can be used by emulator and types - if self.debug: - env['__debug'] = "yes" - # Required for recording source in emulator env['__cdist_manifest'] = manifest_path From cfd04dc20449445ff16e7c3315a091dcebc00d44 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 13:04:01 +0200 Subject: [PATCH 2/6] ++tests needed Signed-off-by: Nico Schottelius --- doc/dev/todo/tests | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/doc/dev/todo/tests b/doc/dev/todo/tests index 27bca3b6..3d17ba34 100644 --- a/doc/dev/todo/tests +++ b/doc/dev/todo/tests @@ -38,20 +38,21 @@ Tests needed for config_install: - Explorers are not transferred twice - No transfer tried if there are no type_explorers - - $whatever should fail if there is no global explorer directory - - emulator may only be called with __ as prefix - fail otherwise! + emulator() + may only be called with __ as prefix - fail otherwise! - - ensure paths returned by object include dot-cdist - - ensure path of explorer of object returns correct path - - config_install: - run_type_manifest() - same tests as for test_initial_manifest_*? - run_manifest() - raise exception if manifest is not existent - object_run(): ensure no object is run twice - object_run(): ensure requirements are taken into account - object_run(): check (from extern?) that all needed variables are setup - object_run(): ensure no code-{local, remote} is created, if gencode is not producing code - object_run(): ensure no code-{local, remote} contains what gencode created + object_run(): + - ensure no object is run twice + - ensure requirements are taken into account? + - and order of run is adjusted + - check (from extern?) that all needed variables are setup + - ensure no code-{local, remote} is created, + if gencode is not producing code + - ensure THAT code-{local, remote} contains what gencode created + - abort if gencode-* fails + - abort if code-* fails + - abort == raise(FooException) + - gencode-*: ensure ENVIRONMENT is setup correctly run_type_explorer(): ensure output is saved to correct path run_type_explorer(): ensure a type with no explorers works @@ -59,3 +60,7 @@ Tests needed for config_install: all: check that messages of all functions contain target_host in string + + config_install: + run_type_manifest() - same tests as for test_initial_manifest_*? + run_manifest() - raise exception if manifest is not existent From 2c5b064b0034a9a4b8aa3be2448adad23f069f78 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 13:06:29 +0200 Subject: [PATCH 3/6] ++todo = remove CODE_HEADER in generated code Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index a286472b..fb27b17d 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -229,6 +229,7 @@ class ConfigInstall: outfile_fd = open(outfile, "w") # Need to flush to ensure our write is done before stdout write + # FIXME: code header still needed? outfile_fd.write(CODE_HEADER) outfile_fd.flush() From 9d7468edf6f7e4408fe04a6d33cdd49018c66efb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 13:09:56 +0200 Subject: [PATCH 4/6] add old log of trigger work Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-06-25.trigger-graphic | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 doc/dev/logs/2011-06-25.trigger-graphic diff --git a/doc/dev/logs/2011-06-25.trigger-graphic b/doc/dev/logs/2011-06-25.trigger-graphic new file mode 100644 index 00000000..f17220e1 --- /dev/null +++ b/doc/dev/logs/2011-06-25.trigger-graphic @@ -0,0 +1,15 @@ + + + + + + +client -------------------> trigger via ssh ----------> server + ~/.ssh/authorized_keys + -> cdist-deploy-to + +ssh -R + +server -> via cdist-deploy-to -> client + +cdist-mass-deploy -p From cfb74897c44d8f3fc05797f0bdb4d83a237c8015 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 13:47:41 +0200 Subject: [PATCH 5/6] write down (almost?) all tests for config_install Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index fb27b17d..418f9fd9 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -192,6 +192,13 @@ class ConfigInstall: cdist.exec.shell_run_or_debug_fail(manifest_path, [manifest_path], env=env) + def object_prepare(self, cdist_object): + """Prepare object: Run type explorer + manifest""" + log.debug("Preparing object: " + cdist_object.name) + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + cdist_object.prepared = True + def object_run(self, cdist_object): """Run gencode and code for an object""" log.debug("Running object %s", cdist_object) @@ -294,7 +301,6 @@ class ConfigInstall: cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=True) output_fd.close() - def link_emulator(self): """Link emulator to types""" src = os.path.abspath(self.exec_path) @@ -302,7 +308,7 @@ class ConfigInstall: dst = os.path.join(self.bin_path, cdist_type.name) log.debug("Linking emulator: %s to %s", src, dst) - # FIXME: handle exception / make it more beautiful + # FIXME: handle exception / make it more beautiful / Steven: raise except :-) os.symlink(src, dst) def run_global_explorers(self): @@ -324,15 +330,6 @@ class ConfigInstall: cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True) output_fd.close() - - def stage_run(self): - """The final (and real) step of deployment""" - log.info("Generating and executing code") - for cdist_object in cdist.core.Object.list_objects(self.object_base_path, - self.type_base_path): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object) - def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" log.info("Deploying to " + self.target_host) @@ -365,12 +362,17 @@ class ConfigInstall: log.debug("Skipping rerun of object %s", cdist_object) continue else: - log.debug("Preparing object: " + cdist_object.name) - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - cdist_object.prepared = True + self.object_prepare(cdist_object) new_objects_created = True + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + for cdist_object in cdist.core.Object.list_objects(self.object_base_path, + self.type_base_path): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object) + def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" src = os.path.join(self.object_base_path, From 2c02b174526089260aa84a93ff99fbacb495e1fe Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 13:47:53 +0200 Subject: [PATCH 6/6] write down (almost?) all tests for config_install (really) Signed-off-by: Nico Schottelius --- doc/dev/todo/tests | 101 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 87 insertions(+), 14 deletions(-) diff --git a/doc/dev/todo/tests b/doc/dev/todo/tests index 3d17ba34..78a63d08 100644 --- a/doc/dev/todo/tests +++ b/doc/dev/todo/tests @@ -26,6 +26,8 @@ Tests needed for config_install: - ENVIRONMENT - they are set - they contain the correct values + - same tests as for test_initial_manifest_*? + run_manifest(): - test all submitted variables: - ENVIRONMENT @@ -33,13 +35,7 @@ Tests needed for config_install: - they are set - they contain the correct values - does $require work? - - transfer_type_explorers(): - - Explorers are not transferred twice - - No transfer tried if there are no type_explorers - - emulator() - may only be called with __ as prefix - fail otherwise! + - check that exception raised, if manifest is not existent object_run(): - ensure no object is run twice @@ -54,13 +50,90 @@ Tests needed for config_install: - abort == raise(FooException) - gencode-*: ensure ENVIRONMENT is setup correctly - run_type_explorer(): ensure output is saved to correct path - run_type_explorer(): ensure a type with no explorers works - run_type_explorer(): ensure environment is setup correctly + run_type_explorer() + - ensure ALL type explores have been run + - ensure output is saved to correct path + - ensure a type with {0,1,2} explorers works ? + - none, one, multiple + - ensure ENVIRONMENT is setup correctly + - fails if ANY of the given explorer fails + + run_global_explorers(): + - ensure ALL type explores have been run + - ensure output is saved to correct path + - ensure a type with {0,1,2} explorers works ? + - none, one, multiple + - ensure ENVIRONMENT is setup correctly + - fails if ANY of the given explorer fails + + link_emulator(): + - ensure that links to ALL types are created + - ensure that links points to correct executable + - i.e. readlink() works + - AND target of readlink is the correct executable + + stage_run() + - ensure that for every object in + cdist.core.Object.list_objects() is passed into + self.object_run() + - instantiate + overwrite + test + + object_prepare(): + - calls (in this order): + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + cdist_object.prepared = True + - object is prepared after function call - all: check that messages of all functions contain target_host in string + stage_prepare(): + - calls (in this order): + self.link_emulator() + self.run_global_explorers() + self.run_initial_manifest() + + - ensure that all objects are created :-) + - as defined in inital manifest + type manifests + + deploy_to() + - ensure self.stage_prepare() and self.stage_run() are + run. [ORDER] + - instantiate + overwrite + test + + deploy_and_cleanup() + - ensure self.deploy_to() + self.cleanup() are are run + run. [ORDER] + - instantiate + overwrite + test + + transfer_object_parameter() + - check that object parameters are transferred + - paths are setup correctly + - test via mock of remote_mkdir and transfer_path + - ensure self.remote_mkdir, self.transfer_path are run. [ORDER] + - instantiate + overwrite + test + + transfer_global_explorers() + - paths are setup correctly + - test via mock of self.remote_mkdir() and self.transfer_path() + - ensure self.remote_mkdir, self.transfer_path are run. [ORDER] + - instantiate + overwrite + test + + transfer_type_explorers(): + - Explorers are not transferred twice + - paths are setup correctly + - test via mock of self.remote_mkdir() and self.transfer_path() + - Explorers are transferred + - test via mock of self.remote_mkdir() and self.transfer_path() + + remote_mkdir() + - is directory created + + remove_remote_path + - is path removed + + transfer_path + - is src to dst transferred? + +emulator: + may only be called with __ as prefix - fail otherwise! - config_install: - run_type_manifest() - same tests as for test_initial_manifest_*? - run_manifest() - raise exception if manifest is not existent