From f8853467f3aae87bd6c1b670aeac75c0dd206bd8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 15:39:51 +0200 Subject: [PATCH 01/12] do not create base directory in transfer_object_parameter, somebody else should have done this Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 9c1f3b9d..734f7280 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -291,10 +291,6 @@ class ConfigInstall: os.path.join(self.context.remote_object_path, cdist_object.parameter_path) - # Create base path before using mkdir -p - # FIXME: needed? - self.remote_mkdir(remote_path) - # Synchronise parameter dir afterwards self.transfer_path(local_path, remote_path) From c1e773199251f79db92f5083ef736b3835056ab7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 15:49:25 +0200 Subject: [PATCH 02/12] cleanups/fixes Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 33 +++++++++------------------------ lib/cdist/context.py | 8 +++++++- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 734f7280..3f1c467e 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -48,15 +48,13 @@ class ConfigInstall: self.exec_path = exec_path self.context = cdist.context.Context(self.target_host, - initial_manifest=initial_manifest, base_path=base_path, + initial_manifest=initial_manifest, + base_path=base_path, debug=debug) - + def cleanup(self): self.path.cleanup() - def __init_env(self): - """Setup environment""" - def run_initial_manifest(self): """Run the initial manifest""" log.info("Running initial manifest %s", self.context.initial_manifest) @@ -248,22 +246,15 @@ class ConfigInstall: self.deploy_to() self.cleanup() - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") +####FIXED ###################################################################### - self.context.remove_remote_path(self.context.remote_base_path) - self.context.remote_mkdir(self.context.remote_base_path) - self.link_emulator() - def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() + self.link_emulator() self.run_global_explorers() self.run_initial_manifest() log.info("Running object manifests and type explorers") - log.debug("Searching for objects in " + cdist.core.Object.base_path()) # Continue process until no new objects are created anymore @@ -281,22 +272,16 @@ class ConfigInstall: cdist_object.prepared = True new_objects_created = True - # FIXME Move into configinstall def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" - local_path = - os.path.join(self.context.object_base_path, - cdist_object.parameter_path) - remote_path = - os.path.join(self.context.remote_object_path, - cdist_object.parameter_path) + src = os.path.join(self.context.object_base_path, + cdist_object.parameter_path) + dst = os.path.join(self.context.remote_object_path, + cdist_object.parameter_path) # Synchronise parameter dir afterwards self.transfer_path(local_path, remote_path) - -####FIXED ###################################################################### - def transfer_global_explorers(self): """Transfer the global explorers""" self.remote_mkdir(self.context.remote_global_explorer_path) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index af37199e..4cfc6c59 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -91,7 +91,7 @@ class Context: # Create directories self.__init_out_paths() - + self.__init_remote_paths() self.__init_env() def cleanup(self): @@ -122,6 +122,12 @@ class Context: os.mkdir(self.global_explorer_out_path) os.mkdir(self.bin_path) + def __init_remote_paths(self): + """Initialise remote directory structure""" + + self.remove_remote_path(self.remote_base_path) + self.remote_mkdir(self.remote_base_path) + def remote_mkdir(self, directory): """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) From e5e2fb866c301474f8e25bd738318cfe1c7882a0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:00:32 +0200 Subject: [PATCH 03/12] ++fixes Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 2 +- lib/cdist/config_install.py | 17 +++++++++-------- lib/cdist/context.py | 17 +++++------------ 3 files changed, 15 insertions(+), 21 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index c5f4b4ef..71ce14ed 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -42,7 +42,7 @@ def config(args): os.environ['__remote_copy'] = "scp -o User=root -q" for host in args.host: - c = Config(host, initial_manifest=args.manifest, base_dir=args.cdist_home, debug=args.debug) + c = Config(host, initial_manifest=args.manifest, base_path=args.cdist_home, debug=args.debug) if args.parallel: log.debug("Creating child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 3f1c467e..c1cdf328 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -70,7 +70,7 @@ class ConfigInstall: log.debug("%s: Running %s", cdist_object.name, manifest) if os.path.exists(manifest_path): env = { "__object" : os.path.join(self.context.object_base_path, - cdist_object.path) + cdist_object.path), "__object_id": cdist_object.object_id, "__object_fq": cdist_object.name, "__type": os.path.join(self.context.type_base_path, @@ -213,16 +213,17 @@ class ConfigInstall: """Run global explorers""" log.info("Running global explorers") - src = cdist.core.GlobalExplorer.base_path - dst = cdist.core.GlobalExplorer.remote_base_path + src_path = cdist.context.global_explorer_path + dst_path = cdist.context.global_explorer_out_path + remote_dst_path = cdist.context.remote_global_explorer_path - self.context.transfer_path(src, dst) + self.context.transfer_path(src_path, remote_dst_path) - for explorer in cdist.core.GlobalExplorer.list_explorers(): - output_fd = open(explorer.out_path, mode='w') + for explorer in os.listdir(src_path): + output_fd = open(os.path.join(dst_path, explorer), mode='w') cmd = [] - cmd.append("__explorer=" + cdist.core.GlobalExplorer.remote_base_path) - cmd.append(explorer.remote_path) + cmd.append("__explorer=" + remote_dst_path) + cmd.append(os.path.join(src_path, explorer)) cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=True) output_fd.close() diff --git a/lib/cdist/context.py b/lib/cdist/context.py index 4cfc6c59..f1f031a8 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -47,11 +47,10 @@ class Context: if base_path: self.base_path = base_path else: - self.base_path = - os.path.abspath( - os.path.join(os.path.dirname(__file__), - os.pardir, - os.pardir)) + self.base_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) # Local input directories @@ -70,7 +69,7 @@ class Context: # Local output directories if out_path: - self.out_path = out_path: + self.out_path = out_path else: self.out_path = os.path.join(tempfile.mkdtemp(), "out") @@ -92,7 +91,6 @@ class Context: # Create directories self.__init_out_paths() self.__init_remote_paths() - self.__init_env() def cleanup(self): # Do not use in __del__: @@ -106,11 +104,6 @@ class Context: shutil.rmtree(self.cache_path) shutil.move(self.base_path, self.cache_path) - #def __init_env(self): - # """Setup environment""" - # os.environ['__cdist_out_path'] = self.out_path - # os.environ['__cdist_base_path'] = self.base_path - def __init_out_paths(self): """Initialise output directory structure""" From 3fcbabc5e8db1f41baf6b706baf1826edd9180a8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:04:37 +0200 Subject: [PATCH 04/12] ++fixes Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 8 ++++---- lib/cdist/context.py | 5 +++++ lib/cdist/copy.py | 13 ------------- 3 files changed, 9 insertions(+), 17 deletions(-) delete mode 100644 lib/cdist/copy.py diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index c1cdf328..78630f25 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -202,7 +202,7 @@ class ConfigInstall: def link_emulator(self): """Link emulator to types""" src = os.path.abspath(self.exec_path) - for type in cdist.core.Type.list_types(): + for type in cdist.core.Type.list_types(self.context.type_base_path): dst = os.path.join(self.context.bin_path, type.name) log.debug("Linking emulator: %s to %s", src, dst) @@ -213,9 +213,9 @@ class ConfigInstall: """Run global explorers""" log.info("Running global explorers") - src_path = cdist.context.global_explorer_path - dst_path = cdist.context.global_explorer_out_path - remote_dst_path = cdist.context.remote_global_explorer_path + src_path = self.context.global_explorer_path + dst_path = self.context.global_explorer_out_path + remote_dst_path = self.context.remote_global_explorer_path self.context.transfer_path(src_path, remote_dst_path) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index f1f031a8..b39e603e 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -128,3 +128,8 @@ class Context: def remove_remote_path(self, destination): cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) + def transfer_path(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_path(destination) + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + ["-r", source, self.target_host + ":" + destination]) diff --git a/lib/cdist/copy.py b/lib/cdist/copy.py deleted file mode 100644 index b0eae54e..00000000 --- a/lib/cdist/copy.py +++ /dev/null @@ -1,13 +0,0 @@ - # FIXME: To Copy - def transfer_dir(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_dir(destination) - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + - ["-r", source, self.target_host + ":" + destination]) - - # FIXME: To Copy - def transfer_file(self, source, destination): - """Transfer file""" - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + - [source, self.target_host + ":" + destination]) - From 9045b30136794c5003c17c0d3307a75601321cb0 Mon Sep 17 00:00:00 2001 From: Thomas Reifenberger Date: Fri, 7 Oct 2011 16:07:02 +0200 Subject: [PATCH 05/12] fix multiple requirements Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index aa22e1fa..8b531e22 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -132,7 +132,7 @@ def run(argv): requirements = os.environ['__require'] log.debug(object_id + ":Writing requirements: " + requirements) require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) + require_fd.write(requirements.replace(" ","\n")) require_fd.close() # Record / Append source From c93e0f90c9062cf6976add1b563828bb770fc29c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:09:10 +0200 Subject: [PATCH 06/12] create remote conf path on new context Signed-off-by: Nico Schottelius --- lib/cdist/context.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index b39e603e..fb00b423 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -85,6 +85,7 @@ class Context: self.remote_conf_path = os.path.join(self.remote_base_path, "conf") self.remote_object_path = os.path.join(self.remote_base_path, "object") + self.remote_type_path = os.path.join(self.remote_conf_path, "type") self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") @@ -120,6 +121,7 @@ class Context: self.remove_remote_path(self.remote_base_path) self.remote_mkdir(self.remote_base_path) + self.remote_mkdir(self.remote_conf_path) def remote_mkdir(self, directory): """Create directory on remote side""" From 14c9317f66384e50888a3fd1baa4f7ee6881658a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:12:15 +0200 Subject: [PATCH 07/12] remove old type.path Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 78630f25..3822b688 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -96,7 +96,7 @@ class ConfigInstall: env['__cdist_manifest'] = manifest_path # Required to find types in emulator - env['__cdist_type_base_path'] = type.path + env['__cdist_type_base_path'] = self.context.type_base_path # Other environment stuff if extra_env: From 2aa0a316e6fad4bebb5c1b819dde1d6722e46614 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:13:00 +0200 Subject: [PATCH 08/12] use _path not _dir in emulator Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 8b531e22..9672d24a 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -30,7 +30,7 @@ log = logging.getLogger(__name__) def run(argv): """Emulate type commands (i.e. __file and co)""" type = os.path.basename(argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + type_dir = os.path.join(os.environ['__cdist_type_base_path'], type) param_dir = os.path.join(type_dir, "parameter") global_dir = os.environ['__global'] object_source = os.environ['__cdist_manifest'] From 149a605b16befdd8c4339af2295e1a4c27639725 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:14:29 +0200 Subject: [PATCH 09/12] use more path, move type -> cdist_type Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 9672d24a..815df948 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -29,9 +29,9 @@ log = logging.getLogger(__name__) def run(argv): """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_path'], type) - param_dir = os.path.join(type_dir, "parameter") + cdist_type = os.path.basename(argv[0]) + type_path = os.path.join(os.environ['__cdist_type_base_path'], cdist_type) + param_dir = os.path.join(type_path, "parameter") global_dir = os.environ['__global'] object_source = os.environ['__cdist_manifest'] @@ -50,14 +50,14 @@ def run(argv): parser.add_argument(argument, action='store', required=True) # If not singleton support one positional parameter - if not os.path.isfile(os.path.join(type_dir, "singleton")): + if not os.path.isfile(os.path.join(type_path, "singleton")): parser.add_argument("object_id", nargs=1) # And finally verify parameter args = parser.parse_args(argv[1:]) # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): + if os.path.isfile(os.path.join(type_path, "singleton")): object_id = "singleton" else: object_id = args.object_id[0] @@ -68,13 +68,13 @@ def run(argv): object_id = object_id[1:] # Prefix output by object_self - logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s' + logformat = '%(levelname)s: ' + cdist_type + '/' + object_id + ': %(message)s' logging.basicConfig(format=logformat) # FIXME: verify object id log.debug(args) - object_dir = os.path.join(global_dir, "object", type, + object_dir = os.path.join(global_dir, "object", cdist_type, object_id, cdist.path.DOT_CDIST) log.debug("Object output dir = " + object_dir) @@ -140,4 +140,4 @@ def run(argv): source_fd.writelines(object_source) source_fd.close() - log.debug("Finished " + type + "/" + object_id + repr(params)) + log.debug("Finished " + cdist_type + "/" + object_id + repr(params)) From 03f5e1b456a1e6bf951965510f7236866ee9f904 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:15:48 +0200 Subject: [PATCH 10/12] fix emulator and cdist module Signed-off-by: Nico Schottelius --- lib/cdist/__init__.py | 2 ++ lib/cdist/emulator.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 6ce0c788..7aca5ac7 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -22,6 +22,8 @@ VERSION = "2.0.3" DOT_CDIST = ".cdist" +import os + class Error(Exception): """Base exception class for this project""" pass diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 815df948..2df95597 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -75,7 +75,7 @@ def run(argv): log.debug(args) object_dir = os.path.join(global_dir, "object", cdist_type, - object_id, cdist.path.DOT_CDIST) + object_id, cdist.DOT_CDIST) log.debug("Object output dir = " + object_dir) param_out_dir = os.path.join(object_dir, "parameter") From a25ef7f0e885cb3ca9bf6d62a14863d7ca30a335 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:16:45 +0200 Subject: [PATCH 11/12] fix emulator and cdist module Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 3822b688..a553f6d8 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -256,7 +256,6 @@ class ConfigInstall: self.run_initial_manifest() log.info("Running object manifests and type explorers") - log.debug("Searching for objects in " + cdist.core.Object.base_path()) # Continue process until no new objects are created anymore new_objects_created = True From ea402fc0c546a79cb73818ea520f6164e1a254ce Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 16:18:38 +0200 Subject: [PATCH 12/12] use new call to object lists in absolute path Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index a553f6d8..2905df67 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -232,7 +232,7 @@ class ConfigInstall: def stage_run(self): """The final (and real) step of deployment""" log.info("Generating and executing code") - for cdist_object in cdist.core.Object.list_objects(): + for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path): log.debug("Run object: %s", cdist_object) self.object_run(cdist_object) @@ -247,8 +247,6 @@ class ConfigInstall: self.deploy_to() self.cleanup() -####FIXED ###################################################################### - def stage_prepare(self): """Do everything for a deploy, minus the actual code stage""" self.link_emulator() @@ -261,7 +259,7 @@ class ConfigInstall: new_objects_created = True while new_objects_created: new_objects_created = False - for cdist_object in cdist.core.Object.list_objects(): + for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path): if cdist_object.prepared: log.debug("Skipping rerun of object %s", cdist_object) continue