From 7823d0c9977aaabf695e38687106d515819fb988 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Oct 2011 11:31:37 +0200 Subject: [PATCH] merge context back into config (it is in fact the same idea) Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 192 ++++++++++++++++++++++++++---------- lib/cdist/context.py | 137 ------------------------- 2 files changed, 139 insertions(+), 190 deletions(-) delete mode 100644 lib/cdist/context.py diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index e4b51ad4..0da820b2 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -24,10 +24,11 @@ import logging import os import stat import sys +import tempfile import time -import cdist.context import cdist.core +import cdist.exec import cdist.emulator log = logging.getLogger(__name__) @@ -37,44 +38,115 @@ CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: """Cdist main class to hold arbitrary data""" - def __init__(self, target_host, initial_manifest=False, - base_path=False, - exec_path=sys.argv[0], - debug=False): + def __init__(self, target_host, + initial_manifest=False, base_path=False, debug=False): self.target_host = target_host - os.environ['target_host'] = target_host self.debug = debug - self.exec_path = exec_path + self.exec_path = sys.argv[0] - self.context = cdist.context.Context(self.target_host, - initial_manifest=initial_manifest, - base_path=base_path, - debug=debug) + # Base and Temp Base + if base_path: + self.base_path = base_path + else: + self.base_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), + os.pardir, + os.pardir)) + + + # Local input + self.cache_path = os.path.join(self.base_path, "cache", + self.target_host) + self.conf_path = os.path.join(self.base_path, "conf") + + self.global_explorer_path = os.path.join(self.conf_path, "explorer") + self.manifest_path = os.path.join(self.conf_path, "manifest") + self.type_base_path = os.path.join(self.conf_path, "type") + self.lib_path = os.path.join(self.base_path, "lib") + + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_path, "init") + + # Local output + if '__cdist_out_dir' in os.environ: + self.out_path = os.environ['__cdist_out_dir'] + else: + self.out_path = os.path.join(tempfile.mkdtemp(), "out") + self.bin_path = os.path.join(self.out_path, "bin") + self.global_explorer_out_path = os.path.join(self.out_path, "explorer") + self.object_base_path = os.path.join(self.out_path, "object") + + # Remote directory base + if '__cdist_remote_out_dir' in os.environ: + self.remote_base_path = os.environ['__cdist_remote_out_dir'] + else: + self.remote_base_path = "/var/lib/cdist" + self.remote_conf_path = os.path.join(self.remote_base_path, "conf") + self.remote_object_path = os.path.join(self.remote_base_path, "object") + + self.remote_type_path = os.path.join(self.remote_conf_path, "type") + self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") + + # Create directories + self.__init_local_paths() + self.__init_remote_paths() + + + def __init_remote_paths(self): + """Initialise remote directory structure""" + self.remove_remote_path(self.remote_base_path) + self.remote_mkdir(self.remote_base_path) + self.remote_mkdir(self.remote_conf_path) + + def __init_local_paths(self): + """Initialise local directory structure""" + + # Create base dir, if user supplied and not existing + if not os.path.isdir(self.base_path): + os.mkdir(self.base_path) + + os.mkdir(self.out_path) + os.mkdir(self.global_explorer_out_path) + os.mkdir(self.bin_path) def cleanup(self): - self.context.cleanup() + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving " + self.out_path + " to " + self.cache_path) + # Remove previous cache + if os.path.exists(self.cache_path): + shutil.rmtree(self.cache_path) + shutil.move(self.out_path, self.cache_path) + + def logfilter(self): + """Add hostname to logs""" def run_initial_manifest(self): """Run the initial manifest""" - log.info("Running initial manifest %s", self.context.initial_manifest) - env = { "__manifest" : self.context.manifest_path } - self.run_manifest(self.context.initial_manifest, extra_env=env) + log.info("Running initial manifest %s", self.initial_manifest) + env = { "__manifest" : self.manifest_path } + self.run_manifest(self.initial_manifest, extra_env=env) def run_type_manifest(self, cdist_object): """Run manifest for a specific object""" cdist_type = cdist_object.type - manifest_path = os.path.join(self.context.type_base_path, + manifest_path = os.path.join(self.type_base_path, cdist_type.manifest_path) log.debug("%s: Running %s", cdist_object.name, manifest_path) if os.path.exists(manifest_path): - env = { "__object" : os.path.join(self.context.object_base_path, + env = { "__object" : os.path.join(self.object_base_path, cdist_object.path), "__object_id": cdist_object.object_id, "__object_fq": cdist_object.name, - "__type": os.path.join(self.context.type_base_path, + "__type": os.path.join(self.type_base_path, cdist_type.path) } self.run_manifest(manifest_path, extra_env=env) @@ -83,11 +155,11 @@ class ConfigInstall: """Run a manifest""" log.debug("Running manifest %s, env=%s", manifest_path, extra_env) env = os.environ.copy() - env['PATH'] = self.context.bin_path + ":" + env['PATH'] + env['PATH'] = self.bin_path + ":" + env['PATH'] # Information required in every manifest env['__target_host'] = self.target_host - env['__global'] = self.context.out_path + env['__global'] = self.out_path # Submit debug flag to manifest, can be used by emulator and types if self.debug: @@ -97,7 +169,7 @@ class ConfigInstall: env['__cdist_manifest'] = manifest_path # Required to find types in emulator - env['__cdist_type_base_path'] = self.context.type_base_path + env['__cdist_type_base_path'] = self.type_base_path # Other environment stuff if extra_env: @@ -124,19 +196,19 @@ class ConfigInstall: # env = os.environ.copy() env['__target_host'] = self.target_host - env['__global'] = self.context.out_path - env["__object"] = os.path.join(self.context.object_base_path, cdist_object.path) + env['__global'] = self.out_path + env["__object"] = os.path.join(self.object_base_path, cdist_object.path) env["__object_id"] = cdist_object.object_id env["__object_fq"] = cdist_object.name env["__type"] = cdist_type.name # gencode for cmd in ["local", "remote"]: - bin = os.path.join(self.context.type_base_path, + bin = os.path.join(self.type_base_path, getattr(cdist_type, "gencode_" + cmd + "_path")) if os.path.isfile(bin): - outfile = os.path.join(self.context.object_base_path, + outfile = os.path.join(self.object_base_path, getattr(cdist_object, "code_" + cmd + "_path")) outfile_fd = open(outfile, "w") @@ -164,12 +236,12 @@ class ConfigInstall: cdist.exec.run_or_fail([code_local]) # code remote - local_remote_code = os.path.join(self.context.object_base_path, + local_remote_code = os.path.join(self.object_base_path, cdist_object.code_remote_path) - remote_remote_code = os.path.join(self.context.remote_object_path, + remote_remote_code = os.path.join(self.remote_object_path, cdist_object.code_remote_path) if os.path.isfile(local_remote_code): - self.context.transfer_path(local_remote_code, remote_remote_code) + self.transfer_path(local_remote_code, remote_remote_code) cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) cdist_object.ran = True @@ -181,12 +253,12 @@ class ConfigInstall: self.transfer_type_explorers(cdist_type) cmd = [] - cmd.append("__explorer=" + self.context.remote_global_explorer_path) + cmd.append("__explorer=" + self.remote_global_explorer_path) cmd.append("__type_explorer=" + os.path.join( - self.context.remote_type_path, + self.remote_type_path, cdist_type.explorer_path)) cmd.append("__object=" + os.path.join( - self.context.remote_object_path, + self.remote_object_path, cdist_object.path)) cmd.append("__object_id=" + cdist_object.object_id) cmd.append("__object_fq=" + cdist_object.name) @@ -195,9 +267,9 @@ class ConfigInstall: self.transfer_object_parameter(cdist_object) for explorer in cdist_type.explorers: - remote_cmd = cmd + [os.path.join(self.context.remote_base_path, + remote_cmd = cmd + [os.path.join(self.remote_base_path, cdist_type.explorer_path, explorer)] - output = os.path.join(self.context.object_base_path, + output = os.path.join(self.object_base_path, cdist_object.explorer_path, explorer) output_fd = open(output, mode='w') log.debug("%s exploring %s using %s storing to %s", @@ -210,8 +282,8 @@ class ConfigInstall: def link_emulator(self): """Link emulator to types""" src = os.path.abspath(self.exec_path) - for cdist_type in cdist.core.Type.list_types(self.context.type_base_path): - dst = os.path.join(self.context.bin_path, cdist_type.name) + for cdist_type in cdist.core.Type.list_types(self.type_base_path): + dst = os.path.join(self.bin_path, cdist_type.name) log.debug("Linking emulator: %s to %s", src, dst) # FIXME: handle exception / make it more beautiful @@ -221,11 +293,11 @@ class ConfigInstall: """Run global explorers""" log.info("Running global explorers") - src_path = self.context.global_explorer_path - dst_path = self.context.global_explorer_out_path - remote_dst_path = self.context.remote_global_explorer_path + src_path = self.global_explorer_path + dst_path = self.global_explorer_out_path + remote_dst_path = self.remote_global_explorer_path - self.context.transfer_path(src_path, remote_dst_path) + self.transfer_path(src_path, remote_dst_path) for explorer in os.listdir(src_path): output_fd = open(os.path.join(dst_path, explorer), mode='w') @@ -240,8 +312,8 @@ class ConfigInstall: def stage_run(self): """The final (and real) step of deployment""" log.info("Generating and executing code") - for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, - self.context.type_base_path): + for cdist_object in cdist.core.Object.list_objects(self.object_base_path, + self.type_base_path): log.debug("Run object: %s", cdist_object) self.object_run(cdist_object) @@ -271,8 +343,8 @@ class ConfigInstall: new_objects_created = True while new_objects_created: new_objects_created = False - for cdist_object in cdist.core.Object.list_objects(self.context.object_base_path, - self.context.type_base_path): + for cdist_object in cdist.core.Object.list_objects(self.object_base_path, + self.type_base_path): if cdist_object.prepared: log.debug("Skipping rerun of object %s", cdist_object) continue @@ -285,19 +357,19 @@ class ConfigInstall: def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" - src = os.path.join(self.context.object_base_path, + src = os.path.join(self.object_base_path, cdist_object.parameter_path) - dst = os.path.join(self.context.remote_object_path, + dst = os.path.join(self.remote_object_path, cdist_object.parameter_path) # Synchronise parameter dir afterwards - self.context.remote_mkdir(dst) - self.context.transfer_path(src, dst) + self.remote_mkdir(dst) + self.transfer_path(src, dst) def transfer_global_explorers(self): """Transfer the global explorers""" - self.remote_mkdir(self.context.remote_global_explorer_path) - self.transfer_path(self.context.global_explorer_path, + self.remote_mkdir(self.remote_global_explorer_path) + self.transfer_path(self.global_explorer_path, self.remote_global_explorer_path) def transfer_type_explorers(self, cdist_type): @@ -313,13 +385,27 @@ class ConfigInstall: if len(explorers) > 0: rel_path = cdist_type.explorer_path - src = os.path.join(self.context.type_base_path, rel_path) - dst = os.path.join(self.context.remote_type_path, rel_path) + src = os.path.join(self.type_base_path, rel_path) + dst = os.path.join(self.remote_type_path, rel_path) # Ensure full path until type exists: # /var/lib/cdist/conf/type/__directory/explorer # /var/lib/cdist/conf/type/__directory may not exist, # but remote_mkdir uses -p to fix this - self.context.remote_mkdir(dst) - self.context.transfer_path(src, dst) + self.remote_mkdir(dst) + self.transfer_path(src, dst) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) + + def remove_remote_path(self, destination): + """Ensure path on remote side vanished""" + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) + + def transfer_path(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_path(destination) + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + ["-r", source, self.target_host + ":" + destination]) diff --git a/lib/cdist/context.py b/lib/cdist/context.py deleted file mode 100644 index a4fdbca5..00000000 --- a/lib/cdist/context.py +++ /dev/null @@ -1,137 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import shutil -import sys -import tempfile - - -log = logging.getLogger(__name__) - -import cdist.exec - -class Context: - """Storing context information""" - - def __init__(self, - target_host, - initial_manifest=False, - base_path=False, - out_path=False, - remote_base_path=False, - debug=False): - - self.target_host = target_host - - # Base and Temp Base - if base_path: - self.base_path = base_path - else: - self.base_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), - os.pardir, - os.pardir)) - - - # Local input directories - self.cache_path = os.path.join(self.base_path, "cache", target_host) - self.conf_path = os.path.join(self.base_path, "conf") - - self.global_explorer_path = os.path.join(self.conf_path, "explorer") - self.manifest_path = os.path.join(self.conf_path, "manifest") - self.type_base_path = os.path.join(self.conf_path, "type") - self.lib_path = os.path.join(self.base_path, "lib") - - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_path, "init") - - # Local output directories - if out_path: - self.out_path = out_path - else: - self.out_path = os.path.join(tempfile.mkdtemp(), "out") - - self.bin_path = os.path.join(self.out_path, "bin") - self.global_explorer_out_path = os.path.join(self.out_path, "explorer") - self.object_base_path = os.path.join(self.out_path, "object") - - # Remote directories - if remote_base_path: - self.remote_base_path = remote_base_path - else: - self.remote_base_path = "/var/lib/cdist" - - self.remote_conf_path = os.path.join(self.remote_base_path, "conf") - self.remote_object_path = os.path.join(self.remote_base_path, "object") - - self.remote_type_path = os.path.join(self.remote_conf_path, "type") - self.remote_global_explorer_path = os.path.join(self.remote_conf_path, "explorer") - - # Create directories - self.__init_out_paths() - self.__init_remote_paths() - - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving " + self.out_path + " to " + self.cache_path) - # Remove previous cache - if os.path.exists(self.cache_path): - shutil.rmtree(self.cache_path) - shutil.move(self.out_path, self.cache_path) - - def __init_out_paths(self): - """Initialise output directory structure""" - - # Create base dir, if user supplied and not existing - if not os.path.isdir(self.base_path): - os.mkdir(self.base_path) - - os.mkdir(self.out_path) - os.mkdir(self.global_explorer_out_path) - os.mkdir(self.bin_path) - - def __init_remote_paths(self): - """Initialise remote directory structure""" - - self.remove_remote_path(self.remote_base_path) - self.remote_mkdir(self.remote_base_path) - self.remote_mkdir(self.remote_conf_path) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) - - def remove_remote_path(self, destination): - cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) - - def transfer_path(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_path(destination) - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + - ["-r", source, self.target_host + ":" + destination])