From 9128cc28c414e1fa1abe4a482f6777f753123910 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 15:50:17 +0200 Subject: [PATCH 1/2] introduce a lot todo in path Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 2dd9dcf1..5cde357b 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -107,6 +107,7 @@ class Path: else: self.initial_manifest = os.path.join(self.manifest_dir, "init") + # FIXME: stays def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -120,13 +121,16 @@ class Path: shutil.move(self.temp_dir, self.cache_dir) + # FIXME: belongs to here - clearify remote* def remote_mkdir(self, directory): """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) + # FIXME: belongs to here - clearify remote* def remove_remote_dir(self, destination): cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) + # FIXME: belongs to here - clearify remote* def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) @@ -135,6 +139,7 @@ class Path: self.target_host + ":" + destination]) + # FIXME: belongs to here - clearify remote* def transfer_file(self, source, destination): """Transfer file""" cdist.exec.run_or_fail(["scp", "-q", source, @@ -142,10 +147,12 @@ class Path: self.target_host + ":" + destination]) + # FIXME: Explorer or stays def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + # FIXME: object def type_explorer_output_dir(self, cdist_object): """Returns and creates dir of the output for a type explorer""" dir = os.path.join(self.object_dir(cdist_object), "explorer") @@ -154,14 +161,17 @@ class Path: return dir + # FIXME Stays here / Explorer? def remote_global_explorer_path(self, explorer): """Returns path to the remote explorer""" return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + # FIXME: stays here def list_global_explorers(self): """Return list of available explorers""" return os.listdir(self.global_explorer_dir) + # FIXME: Type - only needs to know its path def list_type_explorers(self, type): """Return list of available explorers for a specific type""" dir = self.type_dir(type, "explorer") @@ -174,15 +184,18 @@ class Path: return list + # Stays here def list_types(self): """Retuns list of types""" return os.listdir(self.type_base_dir) + # FIXME: type def is_install_type(self, type): """Check whether a type is used for installation (if not: for configuration)""" marker = os.path.join(self.type_dir(type), "install") return os.path.isfile(marker) + # Stays here def list_object_paths(self, starting_point): """Return list of paths of existing objects""" object_paths = [] @@ -198,36 +211,43 @@ class Path: return object_paths - # FIXME + # FIXME: Object def get_type_from_object(self, cdist_object): """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + # FIXME: Object def get_object_id_from_object(self, cdist_object): """Returns everything but the first part (i.e. object_id) of an object""" return os.sep.join(cdist_object.split(os.sep)[1:]) + # FIXME: Object def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + # FIXME: Object def remote_object_dir(self, cdist_object): """Returns the remote full path to the object (including .cdist)""" return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + # FIXME: Object def object_parameter_dir(self, cdist_object): """Returns the dir to the object parameter""" return os.path.join(self.object_dir(cdist_object), "parameter") + # FIXME: object def remote_object_parameter_dir(self, cdist_object): """Returns the remote dir to the object parameter""" return os.path.join(self.remote_object_dir(cdist_object), "parameter") + # FIXME: object def object_code_paths(self, cdist_object): """Return paths to code scripts of object""" return [os.path.join(self.object_dir(cdist_object), "code-local"), os.path.join(self.object_dir(cdist_object), "code-remote")] + # Stays here def list_objects(self): """Return list of existing objects""" @@ -240,14 +260,17 @@ class Path: return objects + # FIXME: Type def type_dir(self, type, *args): - """Return directory the type""" + """Return (sub-)directory of a type""" return os.path.join(self.type_base_dir, type, *args) + # FIXME: Type def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + # Stays here def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" # Create base path before using mkdir -p @@ -257,11 +280,13 @@ class Path: self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) + # Stays here def transfer_global_explorers(self): """Transfer the global explorers""" self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + # Stays here def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" if type in self.type_explorers_transferred: From 28428177ae8734c3b4eb84d361f2c6447c58debc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 15:53:34 +0200 Subject: [PATCH 2/2] --typo Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 426 ++++++++++++++++++------------------ 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index cd8ea720..a550c45e 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -31,263 +31,263 @@ import cdist.path log = logging.getLogger(__name__) -CODE_HEADER = "#!/bin/sh -e\n" +CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: - """Class to hold install and config methods""" + """Class to hold install and config methods""" - def __init__(self, target_host, - initial_manifest=False, - remote_user="root", - home=None, - exec_path=sys.argv[0], - debug=False): + def __init__(self, target_host, + initial_manifest=False, + remote_user="root", + home=None, + exec_path=sys.argv[0], + debug=False): - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user - self.exec_path = exec_path + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.exec_path = exec_path - # FIXME: broken - construct elsewhere! - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + # FIXME: broken - construct elsewhere! + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] - self.path = cdist.path.Path(self.target_host, - initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, - base_dir=home, - debug=debug) - - self.objects_prepared = [] + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] - def cleanup(self): - self.path.cleanup() + def cleanup(self): + self.path.cleanup() - def run_global_explores(self): - """Run global explorers""" - log.info("Running global explorers") - explorers = self.path.list_global_explorers() - if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) + def run_global_explorers(self): + """Run global explorers""" + log.info("Running global explorers") + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) - self.path.transfer_global_explorers() - for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.path.remote_global_explorer_path(explorer)) + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" - type = self.path.get_type_from_object(cdist_object) - self.path.transfer_type_explorers(type) + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) - # Need to transfer at least the parameters for objects to be useful - self.path.transfer_object_parameter(cdist_object) + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) - explorers = self.path.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() - def link_emulator(self): - """Link emulator to types""" - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - self.link_emulator() + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() - def run_initial_manifest(self): - """Run the initial manifest""" - log.info("Running initial manifest %s", self.path.initial_manifest) - env = { "__manifest" : self.path.manifest_dir } - self.run_manifest(self.path.initial_manifest, extra_env=env) + def run_initial_manifest(self): + """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.path.get_type_from_object(cdist_object) - manifest = self.path.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.path.object_dir(cdist_object), - "__object_id": self.path.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.path.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - - # Submit debug flag to manifest, can be used by emulator and types - if self.debug: - env['__debug'] = "yes" + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" - # Required for recording source - env['__cdist_manifest'] = manifest + # Required for recording source + env['__cdist_manifest'] = manifest - # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir - # Other environment stuff - if extra_env: - env.update(extra_env) + # Other environment stuff + if extra_env: + env.update(extra_env) - cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) - if mode == "gencode": - paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) - outfile_fd = open(outfile, "w") + outfile_fd = open(outfile, "w") - # Need to flush to ensure our write is done before stdout write - # FIXME: CODE_HEADER needed in our sh -e scenario???? - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() + # Need to flush to ensure our write is done before stdout write + # FIXME: CODE_HEADER needed in our sh -e scenario? + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() - status = os.stat(outfile) + status = os.stat(outfile) - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - log.info("Running object manifests and type explorers") + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explorers() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") - old_objects = [] - objects = self.path.list_objects() + old_objects = [] + objects = self.path.list_objects() - # Continue process until no new objects are created anymore - while old_objects != objects: - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) + # Continue process until no new objects are created anymore + while old_objects != objects: + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) - objects = self.path.list_objects() + objects = self.path.list_objects() - def stage_run(self): - """The final (and real) step of deployment""" - log.info("Generating and executing code") - # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() - self.stage_prepare() - self.stage_run() + self.stage_prepare() + self.stage_run() - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup()