From 00550edfcbed8f767994ac602846cb7ff634de42 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 12:58:35 +0200 Subject: [PATCH 01/11] finish run_manifest() Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 96f9224e..548d3808 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -83,8 +83,7 @@ class ConfigInstall: # Information required in every manifest env['__target_host'] = self.target_host - # FIXME: __global == __cdist_out_dir - # FIXME: __global? shouldn't this be $global? + # FIXME: __global == __cdist_out_dir - duplication env['__global'] = self.context.out_dir # Submit debug flag to manifest, can be used by emulator and types @@ -95,7 +94,7 @@ class ConfigInstall: env['__cdist_manifest'] = manifest # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir + env['__cdist_type_base_dir'] = type.path # Other environment stuff if extra_env: @@ -104,6 +103,7 @@ class ConfigInstall: cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) ################################################################################ + def object_run(self, cdist_object, mode): """Run gencode or code for an object""" log.debug("Running %s from %s", mode, cdist_object) From af5a99dc9fd7ca6ef10589eee7bcc73c88ed8a17 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:14:47 +0200 Subject: [PATCH 02/11] update object_run() for new interface Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 548d3808..64ad33d0 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -108,11 +108,11 @@ class ConfigInstall: """Run gencode or code for an object""" log.debug("Running %s from %s", mode, cdist_object) - # FIXME: replace with new object interface - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) + requirements = cdist_object.requirements + type = cdist_object.type + # FIXME: ensure objects are not run multiple times! + # FIXME: probably mark objects! for requirement in requirements: log.debug("Object %s requires %s", cdist_object, requirement) self.object_run(requirement, mode=mode) @@ -122,22 +122,23 @@ class ConfigInstall: # env = os.environ.copy() env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) + env['__global'] = self.context.out_dir + env["__object"] = cdist_object.path + env["__object_id"] = cdist_object.object_id + env["__object_fq"] = cdist_object.name + env["__type"] = type.name if mode == "gencode": paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") + type.gencode + type.gencode_remote ] - for bin in paths: + + for cmd in ["local", "remote"]: + bin = getattr(type, "gencode_" + cmd) + if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) + outfile = getattr(cdist_object, "code_" + cmd) outfile_fd = open(outfile, "w") @@ -157,6 +158,7 @@ class ConfigInstall: # Add header and make executable - identically to 0o700 os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + # FIXME: use new interface # Mark object as changed open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() From f121934769959386d3e95e77d37663323bb083e3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:34:22 +0200 Subject: [PATCH 03/11] cleanup object_run() finally Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 99 +++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 55 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 64ad33d0..d77f6d7b 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -104,16 +104,17 @@ class ConfigInstall: ################################################################################ - def object_run(self, cdist_object, mode): + def object_run(self, cdist_object): """Run gencode or code for an object""" log.debug("Running %s from %s", mode, cdist_object) - requirements = cdist_object.requirements + # Catch requirements, which re-call us + if cdist_object.ran: + return + type = cdist_object.type - # FIXME: ensure objects are not run multiple times! - # FIXME: probably mark objects! - for requirement in requirements: + for requirement in cdist_object.requirements: log.debug("Object %s requires %s", cdist_object, requirement) self.object_run(requirement, mode=mode) @@ -128,56 +129,46 @@ class ConfigInstall: env["__object_fq"] = cdist_object.name env["__type"] = type.name - if mode == "gencode": - paths = [ - type.gencode - type.gencode_remote - ] + # gencode + for cmd in ["local", "remote"]: + bin = getattr(type, "gencode_" + cmd) - for cmd in ["local", "remote"]: - bin = getattr(type, "gencode_" + cmd) - - if os.path.isfile(bin): - outfile = getattr(cdist_object, "code_" + cmd) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # FIXME: use new interface - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - + outfile = getattr(cdist_object, "code_" + cmd) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + cdist_object.changed=True + + # code local + code_local = cdist_object.code_local + if os.path.isfile(code_local): + cdist.exec.run_or_fail([code_local]) + + # code remote + local_remote_code = cdist_object.code_remote + remote_remote_code = cdist_object.code_remote_remote + if os.path.isfile(local_remote_code): + self.context.transfer_file(local_remote_code, remote_remote_code) + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) + + cdist_object.ran = True - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) - ### Cleaned / check functions: Round 1 :-) ################################# def run_type_explorer(self, cdist_object): """Run type specific explorers for objects""" @@ -239,11 +230,9 @@ class ConfigInstall: def stage_run(self): """The final (and real) step of deployment""" log.info("Generating and executing code") - # Now do the final steps over the existing objects for cdist_object in cdist.core.Object.list_objects(): log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") + self.object_run(cdist_object) def deploy_to(self): """Mimic the old deploy to: Deploy to one host""" From ca5361afc1df1cdb585f581281e9cc5bccf16868 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:36:51 +0200 Subject: [PATCH 04/11] remote cdist.path, use base_dir Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 2 +- lib/cdist/config_install.py | 5 ++--- lib/cdist/emulator.py | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 942d2917..c5f4b4ef 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -42,7 +42,7 @@ def config(args): os.environ['__remote_copy'] = "scp -o User=root -q" for host in args.host: - c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + c = Config(host, initial_manifest=args.manifest, base_dir=args.cdist_home, debug=args.debug) if args.parallel: log.debug("Creating child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index d77f6d7b..a8ea9043 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -25,10 +25,9 @@ import os import stat import sys -import cdist.emulator -import cdist.path - +import cdist.context import cdist.core +import cdist.emulator log = logging.getLogger(__name__) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 77f26e7f..aa22e1fa 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -24,7 +24,6 @@ import logging import os import cdist -import cdist.path log = logging.getLogger(__name__) From 386e2ca34c2c400a6d848dd0f8bc24d50c4f72ce Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:37:56 +0200 Subject: [PATCH 05/11] make cli work again Signed-off-by: Nico Schottelius --- bin/cdist | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 36dc3d5c..bd6825eb 100755 --- a/bin/cdist +++ b/bin/cdist @@ -115,9 +115,7 @@ if __name__ == "__main__": import cdist import cdist.banner import cdist.config - import cdist.exec import cdist.install - import cdist.path commandline() From 1138d8b439acb2f343a4d4f046882ec10b9dff84 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:44:07 +0200 Subject: [PATCH 06/11] remote_remote_code_remote_remote Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index a8ea9043..e8d179e8 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -101,8 +101,6 @@ class ConfigInstall: cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) -################################################################################ - def object_run(self, cdist_object): """Run gencode or code for an object""" log.debug("Running %s from %s", mode, cdist_object) @@ -161,7 +159,7 @@ class ConfigInstall: # code remote local_remote_code = cdist_object.code_remote - remote_remote_code = cdist_object.code_remote_remote + remote_remote_code = cdist_object.remote_code_remote if os.path.isfile(local_remote_code): self.context.transfer_file(local_remote_code, remote_remote_code) cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) From 2b0a337584b929546e1515824c3b8ccc991ab7ec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:45:19 +0200 Subject: [PATCH 07/11] fix base_dir passing Signed-off-by: Nico Schottelius --- bin/cdist | 1 - lib/cdist/config_install.py | 7 ++++--- lib/cdist/context.py | 7 +++++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index bd6825eb..7d2c035b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -112,7 +112,6 @@ if __name__ == "__main__": import cdist.emulator cdist.emulator.run(sys.argv) else: - import cdist import cdist.banner import cdist.config import cdist.install diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index e8d179e8..407cdcd7 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -37,8 +37,9 @@ class ConfigInstall: """Cdist main class to hold arbitrary data""" def __init__(self, target_host, initial_manifest=False, - exec_path=sys.argv[0], - debug=False): + base_dir=False, + exec_path=sys.argv[0], + debug=False): self.target_host = target_host os.environ['target_host'] = target_host @@ -47,7 +48,7 @@ class ConfigInstall: self.exec_path = exec_path self.context = cdist.context.Context(self.target_host, - initial_manifest=initial_manifest, + initial_manifest=initial_manifest, base_dir=base_dir, debug=debug) def cleanup(self): diff --git a/lib/cdist/context.py b/lib/cdist/context.py index 44f432a8..1615d196 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -33,12 +33,15 @@ import cdist.exec class Context: """Storing context dependent information""" - def __init__(self, target_host, initial_manifest=False, debug=False): + def __init__(self, target_host, initial_manifest=False, base_dir=False, + debug=False): self.target_host = target_host # Base and Temp Base - if "__cdist_base_dir" in os.environ: + if base_dir: + self.base_dir = base_dir + elif "__cdist_base_dir" in os.environ: self.base_dir = os.environ['__cdist_base_dir'] else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) From fc7ae28b02f10109ccde685ccfd04021b7131558 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:45:45 +0200 Subject: [PATCH 08/11] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index bfbdd8c3..b8fcc5b0 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,6 +8,8 @@ - Create GlobalExplorer +- base_dir passing in config/config_install superseeded by __cdist_base_dir? + -------------------------------------------------------------------------------- - insert prefix into logger to distinguish between modules From e03ebcd5c163ca583279ae937c4f837a98ea3f4a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:46:30 +0200 Subject: [PATCH 09/11] require conf_dir to find manifest_dir to find manifest Signed-off-by: Nico Schottelius --- lib/cdist/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index 1615d196..673309af 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -48,10 +48,10 @@ class Context: # Input directories self.cache_dir = os.path.join(self.base_dir, "cache", target_host) + self.conf_dir = os.path.join(self.base_dir, "conf") self.manifest_dir = os.path.join(self.conf_dir, "manifest") # Probably unused paths - # self.conf_dir = os.path.join(self.base_dir, "conf") # self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") # self.lib_dir = os.path.join(self.base_dir, "lib") # self.type_base_dir = os.path.join(self.conf_dir, "type") From fc6ae548893fd9bc1a447e79aec70ba417145057 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:48:10 +0200 Subject: [PATCH 10/11] os.path not os Signed-off-by: Nico Schottelius --- lib/cdist/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/context.py b/lib/cdist/context.py index 673309af..3ecd84e0 100644 --- a/lib/cdist/context.py +++ b/lib/cdist/context.py @@ -106,7 +106,7 @@ class Context: """Initialise output directory structure""" # Create base dir, if user supplied and not existing - if not os.isdir(self.base_dir): + if not os.path.isdir(self.base_dir): os.mkdir(self.base_dir) os.mkdir(self.out_dir) From ed347ffb8a547ca3df02a98b2241c29bc3a20faf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2011 13:49:14 +0200 Subject: [PATCH 11/11] fix init_deploy() Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 407cdcd7..8d926447 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -247,8 +247,8 @@ class ConfigInstall: """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.context.remove_remote_dir(self.context.remote_base_dir) + self.context.remote_mkdir(self.context.remote_base_dir) self.link_emulator() def stage_prepare(self):