From 790deb464502d97c5e84fe37410cd081d22d4977 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 9 Oct 2011 12:49:57 +0200 Subject: [PATCH 01/15] in theory finish the install command Signed-off-by: Nico Schottelius --- lib/cdist/install.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/cdist/install.py b/lib/cdist/install.py index 87714fa8..0f06f5e7 100644 --- a/lib/cdist/install.py +++ b/lib/cdist/install.py @@ -20,14 +20,14 @@ # # -import logging - +import os import cdist.config_install -log = logging.getLogger(__name__) +class Install(cdist.config_install.ConfigInstall): + def __init__(self, *args, **kargs): + """Enhance config install with install support""" + # Setup environ to be used in emulator + os.environ['__install'] = "yes" -def install(args): - """Install remote system""" - process = {} - + super().__init__(*args, **kargs) From 48a9309522d4223c0ae8f91d6cfb08ac591e8086 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 9 Oct 2011 12:50:47 +0200 Subject: [PATCH 02/15] indent/+FIXME Signed-off-by: Nico Schottelius --- bin/cdist | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index c42e540c..61dad1db 100755 --- a/bin/cdist +++ b/bin/cdist @@ -114,11 +114,16 @@ def configinstall(args, mode): time_start = time.time() + # FIXME: do not overwrite, if set! os.environ['__remote_exec'] = "ssh -o User=root -q" os.environ['__remote_copy'] = "scp -o User=root -q" for host in args.host: - c = mode(host, initial_manifest=args.manifest, base_path=args.cdist_home, debug=args.debug) + c = mode(host, + initial_manifest=args.manifest, + base_path=args.cdist_home, + debug=args.debug, + parallel=True) if args.parallel: log.debug("Creating child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) From 7823d0c9977aaabf695e38687106d515819fb988 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 11:31:37 +0200 Subject: [PATCH 03/15] merge context back into config (it is in fact the same idea) Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 192 ++++++++++++++++++++++++++---------- lib/cdist/context.py | 137 ------------------------- 2 files changed, 139 insertions(+), 190 deletions(-) delete mode 100644 lib/cdist/context.py diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index e4b51ad4..0da820b2 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -24,10 +24,11 @@ import logging import os import stat import sys +import tempfile import time -import cdist.context import cdist.core +import cdist.exec import cdist.emulator log = logging.getLogger(__name__) @@ -37,44 +38,115 @@ CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, initial_manifest=False, - base_path=False, - exec_path=sys.argv[0], - debug=False): + def __init__(self, target_host, + initial_manifest=False, base_path=False, debug=False): self.target_host = target_host - os.environ['target_host'] = target_host self.debug = debug - self.exec_path = exec_path + self.exec_path = sys.argv[0] - self.context = cdist.context.Context(self.target_host, - initial_manifest=initial_manifest, - base_path=base_path, - debug=debug) + # Base and Temp Base + if base_path: + self.base_path = base_path + else: + self.base_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) + + + # Local input + self.cache_path = os.path.join(self.base_path, "cache", + self.target_host) + self.conf_path = os.path.join(self.base_path, "conf") + + self.global_explorer_path = os.path.join(self.conf_path, "explorer") + self.manifest_path = os.path.join(self.conf_path, "manifest") + self.type_base_path = os.path.join(self.conf_path, "type") + self.lib_path = os.path.join(self.base_path, "lib") + + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_path, "init") + + # Local output + if '__cdist_out_dir' in os.environ: + self.out_path = os.environ['__cdist_out_dir'] + else: + self.out_path = os.path.join(tempfile.mkdtemp(), "out") + self.bin_path = os.path.join(self.out_path, "bin") + self.global_explorer_out_path = os.path.join(self.out_path, "explorer") + self.object_base_path = os.path.join(self.out_path, "object") + + # Remote directory base + if '__cdist_remote_out_dir' in os.environ: + self.remote_base_path = os.environ['__cdist_remote_out_dir'] + else: + self.remote_base_path = "/var/lib/cdist" + self.remote_conf_path = os.path.join(self.remote_base_path, "conf") + self.remote_object_path = os.path.join(self.remote_base_path, "object") + + self.remote_type_path = os.path.join(self.remote_conf_path, "type") + self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") + + # Create directories + self.__init_local_paths() + self.__init_remote_paths() + + + def __init_remote_paths(self): + """Initialise remote directory structure""" + self.remove_remote_path(self.remote_base_path) + self.remote_mkdir(self.remote_base_path) + self.remote_mkdir(self.remote_conf_path) + + def __init_local_paths(self): + """Initialise local directory structure""" + + # Create base dir, if user supplied and not existing + if not os.path.isdir(self.base_path): + os.mkdir(self.base_path) + + os.mkdir(self.out_path) + os.mkdir(self.global_explorer_out_path) + os.mkdir(self.bin_path) def cleanup(self): - self.context.cleanup() + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving " + self.out_path + " to " + self.cache_path) + # Remove previous cache + if os.path.exists(self.cache_path): + shutil.rmtree(self.cache_path) + shutil.move(self.out_path, self.cache_path) + + def logfilter(self): + """Add hostname to logs""" def run_initial_manifest(self): """Run the initial manifest""" - log.info("Running initial manifest %s", self.context.initial_manifest) - env = { "__manifest" : self.context.manifest_path } - self.run_manifest(self.context.initial_manifest, extra_env=env) + log.info("Running initial manifest %s", self.initial_manifest) + env = { "__manifest" : self.manifest_path } + self.run_manifest(self.initial_manifest, extra_env=env) def run_type_manifest(self, cdist_object): """Run manifest for a specific object""" cdist_type = cdist_object.type - manifest_path = os.path.join(self.context.type_base_path, + manifest_path = os.path.join(self.type_base_path, cdist_type.manifest_path) log.debug("%s: Running %s", cdist_object.name, manifest_path) if os.path.exists(manifest_path): - env = { "__object" : os.path.join(self.context.object_base_path, + env = { "__object" : os.path.join(self.object_base_path, cdist_object.path), "__object_id": cdist_object.object_id, "__object_fq": cdist_object.name, - "__type": os.path.join(self.context.type_base_path, + "__type": os.path.join(self.type_base_path, cdist_type.path) } self.run_manifest(manifest_path, extra_env=env) @@ -83,11 +155,11 @@ class ConfigInstall: """Run a manifest""" log.debug("Running manifest %s, env=%s", manifest_path, extra_env) env = os.environ.copy() - env['PATH'] = self.context.bin_path + ":" + env['PATH'] + env['PATH'] = self.bin_path + ":" + env['PATH'] # Information required in every manifest env['__target_host'] = self.target_host - env['__global'] = self.context.out_path + env['__global'] = self.out_path # Submit debug flag to manifest, can be used by emulator and types if self.debug: @@ -97,7 +169,7 @@ class ConfigInstall: env['__cdist_manifest'] = manifest_path # Required to find types in emulator - env['__cdist_type_base_path'] = self.context.type_base_path + env['__cdist_type_base_path'] = self.type_base_path # Other environment stuff if extra_env: @@ -124,19 +196,19 @@ class ConfigInstall: # env = os.environ.copy() env['__target_host'] = self.target_host - env['__global'] = self.context.out_path - env["__object"] = os.path.join(self.context.object_base_path, cdist_object.path) + env['__global'] = self.out_path + env["__object"] = os.path.join(self.object_base_path, cdist_object.path) env["__object_id"] = cdist_object.object_id env["__object_fq"] = cdist_object.name env["__type"] = cdist_type.name # gencode for cmd in ["local", "remote"]: - bin = os.path.join(self.context.type_base_path, + bin = os.path.join(self.type_base_path, getattr(cdist_type, "gencode_" + cmd + "_path")) if os.path.isfile(bin): - outfile = os.path.join(self.context.object_base_path, + outfile = os.path.join(self.object_base_path, getattr(cdist_object, "code_" + cmd + "_path")) outfile_fd = open(outfile, "w") @@ -164,12 +236,12 @@ class ConfigInstall: cdist.exec.run_or_fail([code_local]) # code remote - local_remote_code = os.path.join(self.context.object_base_path, + local_remote_code = os.path.join(self.object_base_path, cdist_object.code_remote_path) - remote_remote_code = os.path.join(self.context.remote_object_path, + remote_remote_code = os.path.join(self.remote_object_path, cdist_object.code_remote_path) if os.path.isfile(local_remote_code): - self.context.transfer_path(local_remote_code, remote_remote_code) + self.transfer_path(local_remote_code, remote_remote_code) cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) cdist_object.ran = True @@ -181,12 +253,12 @@ class ConfigInstall: self.transfer_type_explorers(cdist_type) cmd = [] - cmd.append("__explorer=" + self.context.remote_global_explorer_path) + cmd.append("__explorer=" + self.remote_global_explorer_path) cmd.append("__type_explorer=" + os.path.join( - self.context.remote_type_path, + self.remote_type_path, cdist_type.explorer_path)) cmd.append("__object=" + os.path.join( - self.context.remote_object_path, + self.remote_object_path, cdist_object.path)) cmd.append("__object_id=" + cdist_object.object_id) cmd.append("__object_fq=" + cdist_object.name) @@ -195,9 +267,9 @@ class ConfigInstall: self.transfer_object_parameter(cdist_object) for explorer in cdist_type.explorers: - remote_cmd = cmd + [os.path.join(self.context.remote_base_path, + remote_cmd = cmd + [os.path.join(self.remote_base_path, cdist_type.explorer_path, explorer)] - output = os.path.join(self.context.object_base_path, + output = os.path.join(self.object_base_path, cdist_object.explorer_path, explorer) output_fd = open(output, mode='w') log.debug("%s exploring %s using %s storing to %s", @@ -210,8 +282,8 @@ class ConfigInstall: def link_emulator(self): """Link emulator to types""" src = os.path.abspath(self.exec_path) - for cdist_type in cdist.core.Type.list_types(self.context.type_base_path): - dst = os.path.join(self.context.bin_path, cdist_type.name) + for cdist_type in cdist.core.Type.list_types(self.type_base_path): + dst = os.path.join(self.bin_path, cdist_type.name) log.debug("Linking emulator: %s to %s", src, dst) # FIXME: handle exception / make it more beautiful @@ -221,11 +293,11 @@ class ConfigInstall: """Run global explorers""" log.info("Running global explorers") - src_path = self.context.global_explorer_path - dst_path = self.context.global_explorer_out_path - remote_dst_path = self.context.remote_global_explorer_path + src_path = self.global_explorer_path + dst_path = self.global_explorer_out_path + remote_dst_path = self.remote_global_explorer_path - self.context.transfer_path(src_path, remote_dst_path) + self.transfer_path(src_path, remote_dst_path) for explorer in os.listdir(src_path): output_fd = open(os.path.join(dst_path, explorer), mode='w') @@ -240,8 +312,8 @@ class ConfigInstall: def stage_run(self): """The final (and real) step of deployment""" log.info("Generating and executing code") - for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, - self.context.type_base_path): + for cdist_object in cdist.core.Object.list_objects(self.object_base_path, + self.type_base_path): log.debug("Run object: %s", cdist_object) self.object_run(cdist_object) @@ -271,8 +343,8 @@ class ConfigInstall: new_objects_created = True while new_objects_created: new_objects_created = False - for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, - self.context.type_base_path): + for cdist_object in cdist.core.Object.list_objects(self.object_base_path, + self.type_base_path): if cdist_object.prepared: log.debug("Skipping rerun of object %s", cdist_object) continue @@ -285,19 +357,19 @@ class ConfigInstall: def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" - src = os.path.join(self.context.object_base_path, + src = os.path.join(self.object_base_path, cdist_object.parameter_path) - dst = os.path.join(self.context.remote_object_path, + dst = os.path.join(self.remote_object_path, cdist_object.parameter_path) # Synchronise parameter dir afterwards - self.context.remote_mkdir(dst) - self.context.transfer_path(src, dst) + self.remote_mkdir(dst) + self.transfer_path(src, dst) def transfer_global_explorers(self): """Transfer the global explorers""" - self.remote_mkdir(self.context.remote_global_explorer_path) - self.transfer_path(self.context.global_explorer_path, + self.remote_mkdir(self.remote_global_explorer_path) + self.transfer_path(self.global_explorer_path, self.remote_global_explorer_path) def transfer_type_explorers(self, cdist_type): @@ -313,13 +385,27 @@ class ConfigInstall: if len(explorers) > 0: rel_path = cdist_type.explorer_path - src = os.path.join(self.context.type_base_path, rel_path) - dst = os.path.join(self.context.remote_type_path, rel_path) + src = os.path.join(self.type_base_path, rel_path) + dst = os.path.join(self.remote_type_path, rel_path) # Ensure full path until type exists: # /var/lib/cdist/conf/type/__directory/explorer # /var/lib/cdist/conf/type/__directory may not exist, # but remote_mkdir uses -p to fix this - self.context.remote_mkdir(dst) - self.context.transfer_path(src, dst) + self.remote_mkdir(dst) + self.transfer_path(src, dst) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) + + def remove_remote_path(self, destination): + """Ensure path on remote side vanished""" + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) + + def transfer_path(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_path(destination) + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + ["-r", source, self.target_host + ":" + destination]) diff --git a/lib/cdist/context.py b/lib/cdist/context.py deleted file mode 100644 index a4fdbca5..00000000 --- a/lib/cdist/context.py +++ /dev/null @@ -1,137 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import shutil -import sys -import tempfile - - -log = logging.getLogger(__name__) - -import cdist.exec - -class Context: - """Storing context information""" - - def __init__(self, - target_host, - initial_manifest=False, - base_path=False, - out_path=False, - remote_base_path=False, - debug=False): - - self.target_host = target_host - - # Base and Temp Base - if base_path: - self.base_path = base_path - else: - self.base_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), - os.pardir, - os.pardir)) - - - # Local input directories - self.cache_path = os.path.join(self.base_path, "cache", target_host) - self.conf_path = os.path.join(self.base_path, "conf") - - self.global_explorer_path = os.path.join(self.conf_path, "explorer") - self.manifest_path = os.path.join(self.conf_path, "manifest") - self.type_base_path = os.path.join(self.conf_path, "type") - self.lib_path = os.path.join(self.base_path, "lib") - - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_path, "init") - - # Local output directories - if out_path: - self.out_path = out_path - else: - self.out_path = os.path.join(tempfile.mkdtemp(), "out") - - self.bin_path = os.path.join(self.out_path, "bin") - self.global_explorer_out_path = os.path.join(self.out_path, "explorer") - self.object_base_path = os.path.join(self.out_path, "object") - - # Remote directories - if remote_base_path: - self.remote_base_path = remote_base_path - else: - self.remote_base_path = "/var/lib/cdist" - - self.remote_conf_path = os.path.join(self.remote_base_path, "conf") - self.remote_object_path = os.path.join(self.remote_base_path, "object") - - self.remote_type_path = os.path.join(self.remote_conf_path, "type") - self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") - - # Create directories - self.__init_out_paths() - self.__init_remote_paths() - - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving " + self.out_path + " to " + self.cache_path) - # Remove previous cache - if os.path.exists(self.cache_path): - shutil.rmtree(self.cache_path) - shutil.move(self.out_path, self.cache_path) - - def __init_out_paths(self): - """Initialise output directory structure""" - - # Create base dir, if user supplied and not existing - if not os.path.isdir(self.base_path): - os.mkdir(self.base_path) - - os.mkdir(self.out_path) - os.mkdir(self.global_explorer_out_path) - os.mkdir(self.bin_path) - - def __init_remote_paths(self): - """Initialise remote directory structure""" - - self.remove_remote_path(self.remote_base_path) - self.remote_mkdir(self.remote_base_path) - self.remote_mkdir(self.remote_conf_path) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) - - def remove_remote_path(self, destination): - cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) - - def transfer_path(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_path(destination) - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + - ["-r", source, self.target_host + ":" + destination]) From d2b042ab7fbaf15790533d0e74b58676961bd052 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 15:14:48 +0200 Subject: [PATCH 04/15] remove parallel arg, always change logoutput Signed-off-by: Nico Schottelius --- bin/cdist | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 61dad1db..50e66697 100755 --- a/bin/cdist +++ b/bin/cdist @@ -122,8 +122,7 @@ def configinstall(args, mode): c = mode(host, initial_manifest=args.manifest, base_path=args.cdist_home, - debug=args.debug, - parallel=True) + debug=args.debug) if args.parallel: log.debug("Creating child process for %s", host) process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) From 9ce31fa84a9995641958b4b6570635289d5ae5f4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 15:37:39 +0200 Subject: [PATCH 05/15] simplify test Signed-off-by: Nico Schottelius --- build.sh | 22 ++++++---------------- 1 file changed, 6 insertions(+), 16 deletions(-) diff --git a/build.sh b/build.sh index 63c380cf..0464defc 100755 --- a/build.sh +++ b/build.sh @@ -126,25 +126,15 @@ case "$1" in | xargs rm -f ;; - t) + test) shift # skip t + set -x + if [ $# -lt 1 ]; then + set -- cdist.test + fi PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ python3 -m unittest "$@" - ;; - - test) - PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ - python3 -m cdist.test - ;; - - test-install) - PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ - python3 -m unittest cdist.test.test_install - ;; - - test-all) - PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ - python3 -m unittest discover lib/cdist/test '*.py' + ;; *) From c81a2925b87d2654d4db7447900959cd8c31c609 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 15:38:38 +0200 Subject: [PATCH 06/15] accept exec_path, setup__target_host Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 15 +++++++++++---- lib/cdist/exec.py | 4 ++-- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 0da820b2..6191b118 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -38,13 +38,21 @@ CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, - initial_manifest=False, base_path=False, debug=False): + def __init__(self, + target_host, + initial_manifest=False, + base_path=False, + exec_path=sys.argv[0], + debug=False): self.target_host = target_host + # FIXME: should this be setup here? + os.environ['__target_host'] = self.target_host self.debug = debug - self.exec_path = sys.argv[0] + + # Required for testing + self.exec_path = exec_path # Base and Temp Base if base_path: @@ -55,7 +63,6 @@ class ConfigInstall: os.pardir, os.pardir)) - # Local input self.cache_path = os.path.join(self.base_path, "cache", self.target_host) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index a9b8d147..63a3dfa3 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -34,7 +34,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): if remote_prefix: remote_prefix = os.environ['__remote_exec'].split() - remote_prefix.append(os.environ['target_host']) + remote_prefix.append(os.environ['__target_host']) args[0][:0] = remote_prefix log.debug("Shell exec cmd: %s", args) @@ -65,7 +65,7 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): def run_or_fail(*args, remote_prefix=False, **kargs): if remote_prefix: remote_prefix = os.environ['__remote_exec'].split() - remote_prefix.append(os.environ['target_host']) + remote_prefix.append(os.environ['__target_host']) args[0][:0] = remote_prefix log.debug("Exec: " + " ".join(*args)) From a8fc8678340c8235953d55710c8f21c5803e0a57 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 15:43:36 +0200 Subject: [PATCH 07/15] ++test integration Signed-off-by: Nico Schottelius --- lib/cdist/test/__init__.py | 3 --- lib/cdist/test/test_install.py | 12 +++++++----- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/cdist/test/__init__.py b/lib/cdist/test/__init__.py index f614fa05..09429470 100644 --- a/lib/cdist/test/__init__.py +++ b/lib/cdist/test/__init__.py @@ -30,9 +30,6 @@ cdist_commands=["banner", "config", "install"] cdist_exec_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../bin/cdist")) -def exec(): - print(cdist_exec_path) - #class UI(unittest.TestCase): # def test_banner(self): # self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) diff --git a/lib/cdist/test/test_install.py b/lib/cdist/test/test_install.py index 9cfae066..d6502b46 100644 --- a/lib/cdist/test/test_install.py +++ b/lib/cdist/test/test_install.py @@ -29,20 +29,22 @@ sys.path.insert(0, os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) import cdist.config - -cdist_exec_path = os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) - +import cdist.test class Install(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() self.init_manifest = os.path.join(self.temp_dir, "manifest") + + os.environ['__remote_exec'] = "ssh -o User=root -q" + os.environ['__remote_copy'] = "scp -o User=root -q" + self.config = cdist.config.Config("localhost", initial_manifest=self.init_manifest, - exec_path=cdist_exec_path) + exec_path=cdist.test.cdist_exec_path) self.config.link_emulator() + ### NEW FOR INSTALL ############################################################ def test_explorer_ran(self): From 1015810391298387b54efc4b58b44347db45673e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 18:31:56 +0200 Subject: [PATCH 08/15] ++testtodo Signed-off-by: Nico Schottelius --- doc/dev/todo/tests | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/tests b/doc/dev/todo/tests index 8f019fed..6fcd1a04 100644 --- a/doc/dev/todo/tests +++ b/doc/dev/todo/tests @@ -26,3 +26,4 @@ Tests needed: run_type_explorer(): ensure environment is setup correctly + all: check that messages of all functions contain target_host in string From 35bfa2fcdc5d7cf53dbfac796913aa01f6415c44 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 18:50:06 +0200 Subject: [PATCH 09/15] finish merge of context and config_install Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 6191b118..f5b33e3f 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -23,6 +23,7 @@ import logging import os import stat +import shutil import sys import tempfile import time @@ -46,8 +47,6 @@ class ConfigInstall: debug=False): self.target_host = target_host - # FIXME: should this be setup here? - os.environ['__target_host'] = self.target_host self.debug = debug @@ -92,12 +91,16 @@ class ConfigInstall: self.remote_base_path = os.environ['__cdist_remote_out_dir'] else: self.remote_base_path = "/var/lib/cdist" + self.remote_conf_path = os.path.join(self.remote_base_path, "conf") self.remote_object_path = os.path.join(self.remote_base_path, "object") self.remote_type_path = os.path.join(self.remote_conf_path, "type") self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") + # Setup env to be used by others + self.__init_env() + # Create directories self.__init_local_paths() self.__init_remote_paths() @@ -120,6 +123,10 @@ class ConfigInstall: os.mkdir(self.global_explorer_out_path) os.mkdir(self.bin_path) + def __init_env(self): + """Environment usable for other stuff""" + os.environ['__target_host'] = self.target_host + def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -274,7 +281,7 @@ class ConfigInstall: self.transfer_object_parameter(cdist_object) for explorer in cdist_type.explorers: - remote_cmd = cmd + [os.path.join(self.remote_base_path, + remote_cmd = cmd + [os.path.join(self.remote_type_path, cdist_type.explorer_path, explorer)] output = os.path.join(self.object_base_path, cdist_object.explorer_path, explorer) @@ -385,6 +392,7 @@ class ConfigInstall: log.debug("Skipping retransfer for explorers of %s", cdist_type) return else: + log.debug("Ensure no retransfer for %s", cdist_type) # Do not retransfer cdist_type.transferred_explorers = True @@ -402,7 +410,6 @@ class ConfigInstall: self.remote_mkdir(dst) self.transfer_path(src, dst) - def remote_mkdir(self, directory): """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) From 92d3925b1cb33cc5d4a5cfbb27eac7ee024e94c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 00:18:32 +0200 Subject: [PATCH 10/15] add filter to prepend hostname in config_install Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index f5b33e3f..0835f972 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -53,6 +53,9 @@ class ConfigInstall: # Required for testing self.exec_path = exec_path + # Configure logging + log.addFilter(self) + # Base and Temp Base if base_path: self.base_path = base_path @@ -106,6 +109,7 @@ class ConfigInstall: self.__init_remote_paths() + def __init_remote_paths(self): """Initialise remote directory structure""" self.remove_remote_path(self.remote_base_path) @@ -139,8 +143,13 @@ class ConfigInstall: shutil.rmtree(self.cache_path) shutil.move(self.out_path, self.cache_path) - def logfilter(self): - """Add hostname to logs""" + def filter(self, record): + """Add hostname to logs via logging Filter""" + + record.msg = self.target_host + ": " + record.msg + + return True + def run_initial_manifest(self): """Run the initial manifest""" From 94cca2e5372336c90246669c54bd0d89b822ea81 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 00:28:53 +0200 Subject: [PATCH 11/15] prepend hostname in emulator as well Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 1 - lib/cdist/emulator.py | 5 +++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 0835f972..7c66d449 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -150,7 +150,6 @@ class ConfigInstall: return True - def run_initial_manifest(self): """Run the initial manifest""" log.info("Running initial manifest %s", self.initial_manifest) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 27a3abb1..b037d63d 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -34,6 +34,7 @@ def run(argv): param_path = os.path.join(type_path, "parameter") global_path = os.environ['__global'] object_source = os.environ['__cdist_manifest'] + target_host = os.environ['__target_host'] if '__debug' in os.environ: logging.root.setLevel(logging.DEBUG) @@ -68,7 +69,7 @@ def run(argv): object_id = object_id[1:] # Prefix output by object_self - logformat = '%(levelname)s: ' + cdist_type + '/' + object_id + ': %(message)s' + logformat = '%(levelname)s: ' + target_host + ": " + cdist_type + '/' + object_id + ': %(message)s' logging.basicConfig(format=logformat) # FIXME: verify object id @@ -103,7 +104,7 @@ def run(argv): value = getattr(args, param) if value: file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) + log.debug(file + " = " + value) # Already exists, verify all parameter are the same if object_exists: From 72a2543470a98563077afb69587ff1db04680d94 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 00:42:03 +0200 Subject: [PATCH 12/15] introduce install support in emulator Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index b037d63d..7141f16d 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -36,10 +36,28 @@ def run(argv): object_source = os.environ['__cdist_manifest'] target_host = os.environ['__target_host'] + # Logsetup - FIXME: add object_fq as soon as setup! + #id = target_host + ": " + cdist_type + '/' + object_id + id = target_host + ": " + # logformat = '%(levelname)s: ' + target_host + ": " + cdist_type + '/' + object_id + ': %(message)s' + logformat = '%(levelname)s: ' + id + ': %(message)s' + logging.basicConfig(format=logformat) + if '__debug' in os.environ: logging.root.setLevel(logging.DEBUG) else: - logging.basicConfig(level=logging.INFO) + logging.root.setLevel(logging.INFO) + + + if '__install' in os.environ: + install = True + else: + install = False + + if install: + if not os.path.isfile(os.path.join(type_path, "install")): + log.debug("Running in install mode, ignoring non install type") + return True parser = argparse.ArgumentParser(add_help=False) @@ -68,9 +86,6 @@ def run(argv): if object_id[0] == '/': object_id = object_id[1:] - # Prefix output by object_self - logformat = '%(levelname)s: ' + target_host + ": " + cdist_type + '/' + object_id + ': %(message)s' - logging.basicConfig(format=logformat) # FIXME: verify object id log.debug(args) @@ -134,11 +149,13 @@ def run(argv): if "require" in os.environ: requirements = os.environ['require'] log.debug(object_id + ":Writing requirements: " + requirements) + # FIXME: handle exception require_fd = open(os.path.join(object_path, "require"), "a") require_fd.write(requirements.replace(" ","\n")) require_fd.close() # Record / Append source + # FIXME: handle exception source_fd = open(os.path.join(object_path, "source"), "a") source_fd.writelines(object_source) source_fd.close() From 425d973dc28c0e3eb8a75f7815feae8399a8977a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 00:46:01 +0200 Subject: [PATCH 13/15] ++changes for 2.0.3 Signed-off-by: Nico Schottelius --- doc/changelog | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index 798c90e0..3540498e 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,7 +1,10 @@ 2.0.3: * Improved logging, added --verbose, by more quiet by default * Bugfix __user: Correct quoting (Steven Armstrong) - * FIXME: Support for __remote_exec and __remote_copy + * Bugfix requirements: Restore original require="" behaviour + * Feature: Initial undocumented support for replacing + the remote exec and remote copy commands + * Feature: Initial undocumented support for installations in core 2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) From 6199a255a49b2bdcb89a31d17583a9d88235d8f9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 11:06:10 +0200 Subject: [PATCH 14/15] add emulator output Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-11.emulator-output | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 doc/dev/logs/2011-10-11.emulator-output diff --git a/doc/dev/logs/2011-10-11.emulator-output b/doc/dev/logs/2011-10-11.emulator-output new file mode 100644 index 00000000..d1f066c5 --- /dev/null +++ b/doc/dev/logs/2011-10-11.emulator-output @@ -0,0 +1,8 @@ +Debug: +DEBUG: Namespace(name=None, state='installed') +DEBUG: Object output dir = /home/users/nico/.tmp/tmpsdaonx/out/object/__package_pacman/zsh/.cdist +DEBUG: Object param dir = /home/users/nico/.tmp/tmpsdaonx/out/object/__package_pacman/zsh/.cdist/parameter +DEBUG: /home/users/nico/.tmp/tmpsdaonx/out/object/__package_pacman/zsh/.cdist/parameter/state<-state = installed +DEBUG: zsh:Writing requirements: +DEBUG: Finished __package_pacman/zsh{'state': 'installed', 'name': None} + From e72ad1f7cb26fb2970117e31154a27560647ecf9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 11 Oct 2011 12:19:13 +0200 Subject: [PATCH 15/15] emulator not needed in config_install anymore Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 7c66d449..2f1adecd 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -30,7 +30,6 @@ import time import cdist.core import cdist.exec -import cdist.emulator log = logging.getLogger(__name__)