diff --git a/doc/dev/todo/steven b/doc/dev/todo/steven index e69de29b..0ff17f6a 100644 --- a/doc/dev/todo/steven +++ b/doc/dev/todo/steven @@ -0,0 +1,8 @@ +Object: + code_remote + code_local + gencode_local + gencode_remote + +Type: + explorer_path diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index fcfa3693..6ce0c788 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -20,6 +20,7 @@ # VERSION = "2.0.3" +DOT_CDIST = ".cdist" class Error(Exception): """Base exception class for this project""" @@ -34,3 +35,17 @@ class MissingEnvironmentVariableError(Error): def __str__(self): return 'Missing required environment variable: ' + str(self.name) + +def file_to_list(filename): + """Return list from \n seperated file""" + if os.path.isfile(filename): + file_fd = open(filename, "r") + lines = file_fd.readlines() + file_fd.close() + + # Remove \n from all lines + lines = map(lambda s: s.strip(), lines) + else: + lines = [] + + return lines diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 2071a0e9..9e2b5edc 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -278,3 +278,39 @@ class ConfigInstall: self.run_type_manifest(cdist_object) cdist_object.prepared = True new_objects_created = True + + # FIXME Move into configinstall + def transfer_object_parameter(self, cdist_object): + """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + + # FIXME Move into configinstall + def transfer_global_explorers(self): + """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + + # FIXME Move into configinstall + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if type.transferred_explorers: + log.debug("Skipping retransfer for explorers of %s", type) + return + else: + # Do not retransfer + type.transferred_explorers = True + + # FIXME: Can be explorer_path or explorer_dir, I don't care. + src = type.explorer_path() + dst = type.remote_explorer_path() + + # Transfer if there is at least one explorer + if len(type.explorers) > 0: + # Ensure that the path exists + self.remote_mkdir(dst) + self.transfer_dir(src, dst) diff --git a/lib/cdist/context.py b/lib/cdist/context.py new file mode 100644 index 00000000..44b121df --- /dev/null +++ b/lib/cdist/context.py @@ -0,0 +1,118 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +import shutil +import sys +import tempfile + + +log = logging.getLogger(__name__) + +import cdist.exec + +class Path: + """Class that handles path related configurations""" + + def __init__(self, target_host, initial_manifest=False, debug=False): + + self.target_host = target_host + + # Base and Temp Base + if "__cdist_base_dir" in os.environ: + self.base_dir = os.environ['__cdist_base_dir'] + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) + + # Input directories + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, target_host) + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_dir, "init") + + # Output directories + if "__cdist_out_dir" in os.environ: + self.out_dir = os.environ['__cdist_out_dir'] + else: + self.out_dir = os.path.join(tempfile.mkdtemp(), "out") + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + self.object_base_dir = os.path.join(self.out_dir, "object") + self.bin_dir = os.path.join(self.out_dir, "bin") + + # Remote directories + if "__cdist_remote_base_dir" in os.environ: + self.remote_base_dir = os.environ['__cdist_remote_base_dir'] + else: + self.remote_base_dir = "/var/lib/cdist" + + self.remote_conf_dir = os.path.join(self.remote_base_dir, "conf") + self.remote_object_dir = os.path.join(self.remote_base_dir, "object") + self.remote_type_dir = os.path.join(self.remote_conf_dir, "type") + self.remote_global_explorer_dir = os.path.join(self.remote_conf_dir, "explorer") + + # Create directories + self.__init_out_dirs() + + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving" + self.base_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + shutil.move(self.base_dir, self.cache_dir) + + def __init_env(self): + """Setup environment""" + os.environ['__cdist_out_dir'] = self.out_dir + + def __init_out_dirs(self): + """Initialise output directory structure""" + + # Create base dir, if user supplied and not existing + if not os.isdir(self.base_dir): + os.mkdir(self.base_dir) + + os.mkdir(self.out_dir) + os.mkdir(self.global_explorer_out_dir) + os.mkdir(self.bin_dir) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) + + def remove_remote_dir(self, destination): + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) + diff --git a/lib/cdist/copy.py b/lib/cdist/copy.py new file mode 100644 index 00000000..b0eae54e --- /dev/null +++ b/lib/cdist/copy.py @@ -0,0 +1,13 @@ + # FIXME: To Copy + def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_dir(destination) + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + ["-r", source, self.target_host + ":" + destination]) + + # FIXME: To Copy + def transfer_file(self, source, destination): + """Transfer file""" + cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + [source, self.target_host + ":" + destination]) + diff --git a/lib/cdist/core/globalexplorer.py b/lib/cdist/core/globalexplorer.py new file mode 100644 index 00000000..49052d53 --- /dev/null +++ b/lib/cdist/core/globalexplorer.py @@ -0,0 +1,66 @@ + # FIXME: Explorer or stays + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + + # FIXME Stays here / Explorer? + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + + # FIXME: stays here + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_dir) + +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +log = logging.getLogger(__name__) + +class Explorer(object): + + def __init__(self, src_dir, dst_dir): + self.src_dir = src_dir + self.dst_dir = dst_dir + + def list_explorers(self): + """Return list of available explorers""" + dir = os.path.join(self.path, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.path, "install")) + + def remote_explorer_dir(self): + """Return remote directory that holds the explorers of a type""" + return os.path.join(self.remote_path, "explorer") diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index dbb6542d..85822a24 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -161,4 +161,75 @@ class Object(object): pass ### /changed - # FIXME: implement other properties/methods + # FIXME: implement other properties/methods + + # FIXME: check following methods: implement or revoke / delete + # FIXME: Object + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + + # FIXME: Object + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + + # FIXME: Object + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + # FIXME: Object + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(self.object_dir(cdist_object), "parameter") + + # FIXME: object + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(self.remote_object_dir(cdist_object), "parameter") + + # FIXME: object + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] + + # Stays here + def list_object_paths(self, starting_point): + """Return list of paths of existing objects""" + object_paths = [] + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + if os.path.isdir(full_path): + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == DOT_CDIST: + object_paths.append(starting_point) + + return object_paths + + # Stays here + def list_objects(self): + """Return list of existing objects""" + + objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) + + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) + + return objects + + # FIXME: object + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_dir(cdist_object), "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + diff --git a/lib/cdist/core/type.py b/lib/cdist/core/type.py index 126b4d16..7492659e 100644 --- a/lib/cdist/core/type.py +++ b/lib/cdist/core/type.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # -# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -68,6 +69,8 @@ class Type(object): self.__required_parameters = None self.__optional_parameters = None + self.transferred_explorers = False + def __repr__(self): return '' % self.name diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 51b2ecc1..f3e9ac30 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -43,10 +43,10 @@ def run(argv): parser = argparse.ArgumentParser(add_help=False) - for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): + for parameter in cdist.file_to_list(os.path.join(param_dir, "optional")): argument = "--" + parameter parser.add_argument(argument, action='store', required=False) - for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): + for parameter in cdist.file_to_list(os.path.join(param_dir, "required")): argument = "--" + parameter parser.add_argument(argument, action='store', required=True) diff --git a/lib/cdist/path.py b/lib/cdist/path.py deleted file mode 100644 index f7d2a8ae..00000000 --- a/lib/cdist/path.py +++ /dev/null @@ -1,273 +0,0 @@ -# -*- coding: utf-8 -*- -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -import logging -import os -import shutil -import sys -import tempfile - -# Hardcoded paths usually not changable -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") -REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") - -DOT_CDIST = ".cdist" - -log = logging.getLogger(__name__) - -import cdist.exec - -def file_to_list(filename): - """Return list from \n seperated file""" - if os.path.isfile(filename): - file_fd = open(filename, "r") - lines = file_fd.readlines() - file_fd.close() - - # Remove \n from all lines - lines = map(lambda s: s.strip(), lines) - else: - lines = [] - - return lines - -class Path: - """Class that handles path related configurations""" - - def __init__(self, - target_host, - initial_manifest=False, - base_dir=None, - debug=False): - - # Base and Temp Base - if base_dir: - self.base_dir = base_dir - else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) - - self.temp_dir = tempfile.mkdtemp() - - self.target_host = target_host - - # Input directories - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, target_host) - self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") - - # Mostly static, but can be overwritten on user demand - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_dir, "init") - - # Output directories - self.out_dir = os.path.join(self.temp_dir, "out") - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - self.object_base_dir = os.path.join(self.out_dir, "object") - self.bin_dir = os.path.join(self.out_dir, "bin") - - os.environ['__cdist_out_dir'] = self.out_dir - - # List of type explorers transferred - self.type_explorers_transferred = {} - - # objects prepared - self.objects_prepared = [] - - # Create directories - self.__init_out_dirs() - - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) - # Remove previous cache - if os.path.exists(self.cache_dir): - shutil.rmtree(self.cache_dir) - shutil.move(self.temp_dir, self.cache_dir) - - - def __init_out_dirs(self): - """Initialise output directory structure""" - os.mkdir(self.out_dir) - os.mkdir(self.global_explorer_out_dir) - os.mkdir(self.bin_dir) - - - # Stays here - def list_types(self): - """Retuns list of types""" - return os.listdir(self.type_base_dir) - - ###################################################################### - - # FIXME: belongs to here - clearify remote* - def remote_mkdir(self, directory): - """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) - - # FIXME: belongs to here - clearify remote* - def remove_remote_dir(self, destination): - cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) - - # FIXME: belongs to here - clearify remote* - def transfer_dir(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_dir(destination) - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + - ["-r", source, self.target_host + ":" + destination]) - - # FIXME: belongs to here - clearify remote* - def transfer_file(self, source, destination): - """Transfer file""" - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + - [source, self.target_host + ":" + destination]) - - # FIXME: Explorer or stays - def global_explorer_output_path(self, explorer): - """Returns path of the output for a global explorer""" - return os.path.join(self.global_explorer_out_dir, explorer) - - # FIXME: object - def type_explorer_output_dir(self, cdist_object): - """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_dir(cdist_object), "explorer") - if not os.path.isdir(dir): - os.mkdir(dir) - - return dir - - # FIXME Stays here / Explorer? - def remote_global_explorer_path(self, explorer): - """Returns path to the remote explorer""" - return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) - - # FIXME: stays here - def list_global_explorers(self): - """Return list of available explorers""" - return os.listdir(self.global_explorer_dir) - - # Stays here - def list_object_paths(self, starting_point): - """Return list of paths of existing objects""" - object_paths = [] - - for content in os.listdir(starting_point): - full_path = os.path.join(starting_point, content) - if os.path.isdir(full_path): - object_paths.extend(self.list_object_paths(starting_point = full_path)) - - # Directory contains .cdist -> is an object - if content == DOT_CDIST: - object_paths.append(starting_point) - - return object_paths - - # FIXME: Object - def get_object_id_from_object(self, cdist_object): - """Returns everything but the first part (i.e. object_id) of an object""" - return os.sep.join(cdist_object.split(os.sep)[1:]) - - # FIXME: Object - def object_dir(self, cdist_object): - """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) - - # FIXME: Object - def remote_object_dir(self, cdist_object): - """Returns the remote full path to the object (including .cdist)""" - return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) - - # FIXME: Object - def object_parameter_dir(self, cdist_object): - """Returns the dir to the object parameter""" - return os.path.join(self.object_dir(cdist_object), "parameter") - - # FIXME: object - def remote_object_parameter_dir(self, cdist_object): - """Returns the remote dir to the object parameter""" - return os.path.join(self.remote_object_dir(cdist_object), "parameter") - - # FIXME: object - def object_code_paths(self, cdist_object): - """Return paths to code scripts of object""" - return [os.path.join(self.object_dir(cdist_object), "code-local"), - os.path.join(self.object_dir(cdist_object), "code-remote")] - - # Stays here - def list_objects(self): - """Return list of existing objects""" - - objects = [] - if os.path.isdir(self.object_base_dir): - object_paths = self.list_object_paths(self.object_base_dir) - - for path in object_paths: - objects.append(os.path.relpath(path, self.object_base_dir)) - - return objects - - # Stays here - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - - # Stays here - def transfer_global_explorers(self): - """Transfer the global explorers""" - self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) - self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - - # Stays here - FIXME: adjust to type code, loop over types! - def transfer_type_explorers(self, type): - """Transfer explorers of a type, but only once""" - if type.transferred: - log.debug("Skipping retransfer for explorers of %s", type) - return - else: - # Do not retransfer - type.transferred = True - - # FIXME: Can be explorer_path or explorer_dir, I don't care. - src = type.explorer_path() - dst = type.remote_explorer_path() - - # Transfer if there is at least one explorer - if len(type.explorers) > 0: - # Ensure that the path exists - self.remote_mkdir(dst) - self.transfer_dir(src, dst)