diff --git a/doc/dev/logs/2011-10-14.error-output b/doc/dev/logs/2011-10-14.error-output new file mode 100644 index 00000000..4414de13 --- /dev/null +++ b/doc/dev/logs/2011-10-14.error-output @@ -0,0 +1,46 @@ +[23:24] brief:cdist% ./bin/cdist config -c ~/p/cdist-nutzung -v ikq04.ethz.ch +INFO: ikq04.ethz.ch: Deploying to ikq04.ethz.ch +INFO: ikq04.ethz.ch: Running object manifests and type explorers +cat: /home/users/nico/.tmp/tmpf969y2/out/object/__addifnosuchline/ssh-root-blukas/.cdist/parameter/line: No such file or directory +#!/bin/sh +# +# 2010-2011 Daniel Roth (dani-cdist@d-roth.li) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +if [ -f "$__object/parameter/file" ]; then + file=$(cat "$__object/parameter/file") +else + file="/$__object_id" +fi + +regex=$(cat "$__object/parameter/line") +if [ -f "$file" ]; then + grep -q "^$regex\$" "$file" + if [ $? -eq 1 ]; then + echo "NOTFOUND" + else + echo "FOUND" + fi +else + echo "NOTFOUND" +fi +ERROR: ikq04.ethz.ch: Code that raised the error: +None +ERROR: Remote script execution failed: /var/lib/cdist/conf/type/__addifnosuchline/explorer/findline ['ssh', '-o', 'User=root', '-q', 'ikq04.ethz.ch', '__explorer=/var/lib/cdist/conf/explorer', '__object_fq=__addifnosuchline/ssh-root-blukas/.cdist', '__target_host=ikq04.ethz.ch', '__object_id=ssh-root-blukas', '__type_explorer=/var/lib/cdist/conf/type/__addifnosuchline/explorer', '__object=/home/users/nico/.tmp/tmpf969y2/out/object/__addifnosuchline/ssh-root-blukas/.cdist', '/bin/sh', '-e', '/var/lib/cdist/conf/type/__addifnosuchline/explorer/findline'] +[23:25] brief:cdist% diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py deleted file mode 100644 index a2282347..00000000 --- a/lib/cdist/exec.py +++ /dev/null @@ -1,96 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import subprocess - -import cdist - - -class Wrapper(object): - def __init__(self, target_host, remote_exec, remote_copy): - self.target_host = target_host - self.remote_exec = remote_exec - self.remote_copy = remote_copy - self.log = logging.getLogger(self.target_host) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) - - def remove_remote_path(self, destination): - """Ensure path on remote side vanished""" - self.run_or_fail(["rm", "-rf", destination], remote=True) - - def transfer_path(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_path(destination) - self.run_or_fail(self.remote_copy.split() + - ["-r", source, self.target_host + ":" + destination]) - - def shell_run_or_debug_fail(self, script, *args, remote=False, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - if remote: - remote_prefix = self.remote_exec.split() - remote_prefix.append(self.target_host) - args[0][:0] = remote_prefix - - self.log.debug("Shell exec cmd: %s", args) - - if 'env' in kargs: - self.log.debug("Shell exec env: %s", kargs['env']) - - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - self.log.error("Code that raised the error:\n") - - if remote: - self.run_or_fail(["cat", script], remote=remote) - - else: - try: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - except IOError as error: - raise cdist.Error(str(error)) - - raise cdist.Error("Command failed (shell): " + " ".join(*args)) - except OSError as error: - raise cdist.Error(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, remote=False, **kargs): - if remote: - remote_prefix = self.remote_exec.split() - remote_prefix.append(self.target_host) - args[0][:0] = remote_prefix - - self.log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - raise cdist.Error("Command failed: " + " ".join(*args)) - except OSError as error: - raise cdist.Error(" ".join(*args) + ": " + error.args[1]) diff --git a/lib/cdist/manifest.py b/lib/cdist/manifest.py deleted file mode 100644 index ab5f7284..00000000 --- a/lib/cdist/manifest.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import stat -import shutil -import sys -import tempfile -import time - -import cdist.core -import cdist.exec - -CODE_HEADER = "#!/bin/sh -e\n" - -class ConfigInstall: - """Cdist main class to hold arbitrary data""" - - def __init__(self, - target_host, - initial_manifest=False, - base_path=False, - exec_path=sys.argv[0], - debug=False): - - self.context = cdist.context.Context( - target_host=target_host, - initial_manifest=initial_manifest, - base_path=base_path, - exec_path=sys.argv[0], - debug=debug) - - - - self.exec_wrapper = cdist.exec.Wrapper( - targe_host = self.target_host, - remote_exec=os.environ['__remote_exec'].split(), - remote_copy=os.environ['__remote_copy'].split() - ) - - self.log = logging.getLogger(self.context.target_host) - - # Setup env to be used by others - FIXME - self.__init_env() - - # Create directories - self.__init_local_paths() - self.__init_remote_paths() - - def __init_remote_paths(self): - """Initialise remote directory structure""" - self.remove_remote_path(self.context.remote_base_path) - self.remote_mkdir(self.context.remote_base_path) - self.remote_mkdir(self.context.remote_conf_path) - - def __init_local_paths(self): - """Initialise local directory structure""" - - # Create base dir, if user supplied and not existing - if not os.path.isdir(self.context.base_path): - os.mkdir(self.context.base_path) - - # FIXME: raise more beautiful exception / Steven: handle exception - os.mkdir(self.out_path) - os.mkdir(self.global_explorer_out_path) - os.mkdir(self.bin_path) - - # FIXME: remove this function, only expose ENV - # explicitly! - def __init_env(self): - """Environment usable for other stuff""" - os.environ['__target_host'] = self.target_host - if self.debug: - os.environ['__debug'] = "yes" - - def cleanup(self): - self.context.cleanup() - - def run_initial_manifest(self): - """Run the initial manifest""" - log.info("Running initial manifest %s", self.initial_manifest) - env = { "__manifest" : self.manifest_path } - self.run_manifest(self.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - cdist_type = cdist_object.type - manifest_path = os.path.join(self.type_base_path, - cdist_type.manifest_path) - - log.debug("%s: Running %s", cdist_object.name, manifest_path) - if os.path.exists(manifest_path): - env = { "__object" : os.path.join(self.object_base_path, - cdist_object.path), - "__object_id": cdist_object.object_id, - "__object_fq": cdist_object.name, - "__type": os.path.join(self.type_base_path, - cdist_type.path) - } - self.run_manifest(manifest_path, extra_env=env) - - def run_manifest(self, manifest_path, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest_path, extra_env) - env = os.environ.copy() - env['PATH'] = self.bin_path + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.out_path - - # Required for recording source in emulator - env['__cdist_manifest'] = manifest_path - - # Required to find types in emulator - env['__cdist_type_base_path'] = self.type_base_path - - # Other environment stuff - if extra_env: - env.update(extra_env) - - cdist.exec.shell_run_or_debug_fail(manifest_path, [manifest_path], env=env) - - def object_prepare(self, cdist_object): - """Prepare object: Run type explorer + manifest""" - log.debug("Preparing object: " + cdist_object.name) - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - cdist_object.prepared = True - - def object_run(self, cdist_object): - """Run gencode and code for an object""" - log.debug("Running object %s", cdist_object) - - # Catch requirements, which re-call us - if cdist_object.ran: - return - - cdist_type = cdist_object.type - - for requirement in cdist_object.requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.out_path - env["__object"] = os.path.join(self.object_base_path, cdist_object.path) - env["__object_id"] = cdist_object.object_id - env["__object_fq"] = cdist_object.name - env["__type"] = cdist_type.name - - # gencode - for cmd in ["local", "remote"]: - bin = os.path.join(self.type_base_path, - getattr(cdist_type, "gencode_" + cmd + "_path")) - - if os.path.isfile(bin): - outfile = os.path.join(self.object_base_path, - getattr(cdist_object, "code_" + cmd + "_path")) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - # FIXME: code header still needed? - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - cdist_object.changed=True - - # code local - code_local = cdist_object.code_local_path - if os.path.isfile(code_local): - cdist.exec.run_or_fail([code_local]) - - # code remote - local_remote_code = os.path.join(self.object_base_path, - cdist_object.code_remote_path) - remote_remote_code = os.path.join(self.remote_object_path, - cdist_object.code_remote_path) - if os.path.isfile(local_remote_code): - self.transfer_path(local_remote_code, remote_remote_code) - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=True) - - cdist_object.ran = True - - def link_emulator(self): - """Link emulator to types""" - src = os.path.abspath(self.exec_path) - for cdist_type in cdist.core.Type.list_types(self.type_base_path): - dst = os.path.join(self.bin_path, cdist_type.name) - log.debug("Linking emulator: %s to %s", src, dst) - - # FIXME: handle exception / make it more beautiful / Steven: raise except :-) - os.symlink(src, dst) - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - self.stage_prepare() - self.stage_run() - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - start_time = time.time() - self.deploy_to() - self.cleanup() - log.info("Finished run of %s in %s seconds", - self.target_host, time.time() - start_time) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.link_emulator() - self.run_global_explorers() - self.run_initial_manifest() - - log.info("Running object manifests and type explorers") - - # Continue process until no new objects are created anymore - new_objects_created = True - while new_objects_created: - new_objects_created = False - for cdist_object in cdist.core.Object.list_objects(self.object_base_path, - self.type_base_path): - if cdist_object.prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.object_prepare(cdist_object) - new_objects_created = True - - def stage_run(self): - """The final (and real) step of deployment""" - log.info("Generating and executing code") - for cdist_object in cdist.core.Object.list_objects(self.object_base_path, - self.type_base_path): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object) diff --git a/lib/cdist/type_explorer.py b/lib/cdist/type_explorer.py deleted file mode 100644 index 9e3c99bf..00000000 --- a/lib/cdist/type_explorer.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import io -import logging -import os -#import stat -#import shutil -#import sys -#import tempfile -#import time -# -#import cdist.exec - -import cdist - -# FIXME: Logging with hostname -log = logging.getLogger(__name__) - -class TypeExplorer(object): - def __init__(self, - remote_global_explorer_path, - object_base_path, - type_base_path, - remote_object_base_path, - remote_type_base_path - ): - - self.object_base_path = object_base_path - self.global_explorer_path = global_explorer_path - self.type_base_path = type_base_path - self.remote_type_base_path = remote_type_base_path - self.remote_object_path = remote_object_path - - def run(self, cdist_object): - """Run type specific explorers for objects""" - - cdist_type = cdist_object.type - - cmd = [] - cmd.append("__explorer=" + self.remote_global_explorer_path) - cmd.append("__type_explorer=" + os.path.join( - self.remote_type_path, - cdist_type.explorer_path)) - cmd.append("__object=" + os.path.join( - self.remote_object_base_path, - cdist_object.path)) - cmd.append("__object_id=" + cdist_object.object_id) - cmd.append("__object_fq=" + cdist_object.name) - - outputs = {} - for explorer in cdist_type.explorers: - remote_cmd = cmd + [os.path.join(self.remote_type_path, - cdist_type.explorer_path, explorer)] - outputs[explorer] = io.StringIO() - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - # FIXME: change to new style - cdist.exec.run_or_fail(remote_cmd, stdout=outputs[explorer], - remote_prefix=True) - - return outputs - - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - src = os.path.join(self.object_base_path, - cdist_object.parameter_path) - dst = os.path.join(self.remote_object_base_path, - cdist_object.parameter_path) - - # FIXME: new style - # Synchronise parameter dir afterwards - self.remote_mkdir(dst) - self.transfer_path(src, dst) - - def transfer_type_explorers(self, cdist_type): - """Transfer explorers of a type, but only once""" - if cdist_type.transferred_explorers: - log.debug("Skipping retransfer for explorers of %s", cdist_type) - return - else: - log.debug("Ensure no retransfer for %s", cdist_type) - # Do not retransfer - cdist_type.transferred_explorers = True - - explorers = cdist_type.explorers - - if len(explorers) > 0: - rel_path = cdist_type.explorer_path - src = os.path.join(self.type_base_path, rel_path) - dst = os.path.join(self.remote_type_path, rel_path) - - # Ensure full path until type exists: - # /var/lib/cdist/conf/type/__directory/explorer - # /var/lib/cdist/conf/type/__directory may not exist, - # but remote_mkdir uses -p to fix this - self.remote_mkdir(dst) - self.transfer_path(src, dst)