diff --git a/cdist/conf/library/manifest b/cdist/conf/library/manifest new file mode 100644 index 00000000..2592beec --- /dev/null +++ b/cdist/conf/library/manifest @@ -0,0 +1,103 @@ +# vi: set filetype=sh: +# $__library/manifest +# +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# This library contains: +# common helper functions helping writing the manifest + + +# check if type is a singleton +# +# Arguments: +# 1: name of the type, e.g. `__some_type` +is_singleton() { + if [ -f "$__global/conf/type/$1/singleton" ]; then + return 0 + else + return 1 + fi +} + +# print object id +# +# Arguments: +# 1: type name, e.g. `__some_type` +# 2: object id; will be ignored if type is a singleton +get_object_id() { + if is_singleton "$1"; then + printf "%s" "$1" + else + printf "%s/%s" "$1" "$2" + fi +} + + +# Helper function to get the correct chain name. It will shorten any +# invalid character. +# +# Arguments: +# 1: the dependency chain name +_rec_normalize_chain_name() { + # spaces will be underscores + # anything other than a-zA-Z0-9 will be removed + printf "$1" | sed 's/\([[:space:]]\|-\)/_/g; s/[^a-zA-Z0-9_]//g' +} + + +# Recording dependencies +# +# Arguments: +# 1: the dependency chain - only chars, numbers and underscores allowed +# 2..n: the type command; no more nesting possible +rec_requires() { + # make it ready to be passed to eval + _rec_chain="$(_rec_normalize_chain_name "$1")" + eval "_rec_ch_${_rec_chain}=\"\${_rec_ch_${_rec_chain}:-} $(get_object_id "$2" "$3")\"" + + # execute type + shift + "$@" +} + + +# Clearing dependency chains +# +# Arguments: +# 1: the dependency chain name +rec_clear() { + _rec_chain="$(printf "$1" | sed 's/\([[:space:]]\|-\)/_/g; s/[^a-zA-Z0-9_]//g')" + eval "unset -v _rec_ch_${_rec_chain}" +} + + +# Export dependencies to current command by appending to the current environment +# +# Arguments: +# 1: the dependency chain +# 2..n: the type command; no more nesting possible +depend() { + # assemble dependency + _rec_chain="$(_rec_normalize_chain_name "$1")" + _rec_deps="$(eval "printf \"%s\" \"\${_rec_ch_${_rec_chain}:-}\"")" + + # execute type + shift + require="$require $_rec_deps" "$@" +} diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index b68448bc..7807d24b 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -56,8 +56,10 @@ class CdistType: self.absolute_path = os.path.join(self.base_path, self.path) if not os.path.isdir(self.absolute_path): raise InvalidTypeError(self.name, self.path, self.absolute_path) + self.manifest_path = os.path.join(self.name, "manifest") self.explorer_path = os.path.join(self.name, "explorer") + self.library_path = os.path.join(self.name, "library") self.gencode_local_path = os.path.join(self.name, "gencode-local") self.gencode_remote_path = os.path.join(self.name, "gencode-remote") self.manifest_path = os.path.join(self.name, "manifest") @@ -71,6 +73,8 @@ class CdistType: self.__parameter_defaults = None self.__deprecated_parameters = None + self._transferred_type_library = False + def __hash__(self): return hash(self.name) @@ -296,3 +300,25 @@ class CdistType: finally: self.__deprecated_parameters = deprecated return self.__deprecated_parameters + + # transfer of the library required by type explorers and code-remote + def transfer_type_library(self, target): + """Transfer the type library for the given type to the remote side. + + target: Code | Explorer | Manifest + (must contain .local and .remote properties) + """ + if not self._transferred_type_library: + source = os.path.join(target.local.type_path, + self.library_path) + destination = os.path.join(target.remote.type_path, + self.library_path) + + if os.path.isdir(source) and os.listdir(source): + target.remote.transfer(source, destination) + target.remote.run(["chmod", "0700", "%s/*" % (destination)]) + else: + # at least, the directory should exist + target.remote.mkdir(destination) + + self._transferred_type_library = True diff --git a/cdist/core/code.py b/cdist/core/code.py index 12888ba4..e86b5050 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -39,6 +39,8 @@ common: __cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator == local.type_path + __library: full qualified path to the library folder containing + common code == local.global_library_path gencode-local script: full qualified path to a types gencode-local @@ -107,6 +109,7 @@ class Code: '__target_fqdn': self.target_host[2], '__global': self.local.base_path, '__files': self.local.files_path, + '__library': self.local.global_library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(local.log), '__cdist_log_level_name': util.log_level_name_env_var_val( @@ -157,7 +160,12 @@ class Code: def transfer_code_remote(self, cdist_object): """Transfer the code_remote script for the given object to the - remote side.""" + remote side. Will transfer the type library too if not already + copied to the remote side cause of type explorers.""" + # $__type/library (if not already done) + cdist_object.cdist_type.transfer_type_library(self) + + # $__object/code-remote source = os.path.join(self.local.object_path, cdist_object.code_remote_path) destination = os.path.join(self.remote.object_path, @@ -188,6 +196,7 @@ class Code: env = os.environ.copy() env.update(self.env) env.update({ + '__type': cdist_object.cdist_type.absolute_path, '__object': cdist_object.absolute_path, '__object_id': cdist_object.object_id, }) @@ -196,11 +205,17 @@ class Code: def run_code_remote(self, cdist_object): """Run the code-remote script for the given cdist object on the remote side.""" - # Put some env vars, to allow read only access to the parameters - # over $__object which is already on the remote side env = { + # Put some env vars, to allow read only access to the parameters + # over $__object which is already on the remote side '__object': os.path.join(self.remote.object_path, cdist_object.path), '__object_id': cdist_object.object_id, + + # The libraries should be over there, too + '__library': self.remote.global_library_path, + '__type_library': os.path.join( + self.remote.type_path, + cdist_object.cdist_type.library_path), } return self._run_code(cdist_object, 'remote', env=env) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 48414ade..aa6bae46 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -36,6 +36,9 @@ common: __explorer: full qualified path to other global explorers on remote side -> remote.global_explorer_path + __library: full qualified path to global library files on the + remote side + -> remote.global_library_path a global explorer is: - a script @@ -57,6 +60,7 @@ type explorer is: __object_fq: full qualified object id, iow: $type.name + / + object_id __type_explorer: full qualified path to the other type explorers on remote side + __type_library: full qualified path to the type library files on remote creates: nothing, returns output @@ -79,6 +83,7 @@ class Explorer: '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__explorer': self.remote.global_explorer_path, + '__library': self.remote.global_library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( @@ -106,6 +111,7 @@ class Explorer: """ self.log.verbose("Running global explorers") + self.transfer_global_library() self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) @@ -163,6 +169,14 @@ class Explorer: self.remote.run(["chmod", "0700", "{}/*".format( self.remote.global_explorer_path)]) + def transfer_global_library(self): + """Transfer the global library files to the remote side.""" + self.remote.transfer(self.local.global_library_path, + self.remote.global_library_path, + self.jobs) + self.remote.run(["chmod", "0700", + "%s/*" % (self.remote.global_library_path)]) + def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" script = os.path.join(self.remote.global_explorer_path, explorer) @@ -223,7 +237,9 @@ class Explorer: '__object_name': cdist_object.name, '__object_fq': cdist_object.path, '__type_explorer': os.path.join(self.remote.type_path, - cdist_type.explorer_path) + cdist_type.explorer_path), + '__type_library': os.path.join(self.remote.type_path, + cdist_type.library_path), }) script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer) @@ -237,6 +253,10 @@ class Explorer: self.log.trace("Skipping retransfer of type explorers for: %s", cdist_type) else: + # transfer library by common code + cdist_type.transfer_type_library(self) + + # transfer type explorers source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 09e74dac..7352ea2b 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -43,6 +43,7 @@ common: types are defined for use in type emulator == local.type_path __files: full qualified path to the files dir + __library: full qualitfied path to the library dir __target_host_tags: comma spearated list of host tags initial manifest is: @@ -114,6 +115,7 @@ class Manifest: '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__files': self.local.files_path, + '__library': self.local.global_library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 370336ad..5b16fab6 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -36,7 +36,7 @@ import cdist.message from cdist import core import cdist.exec.util as util -CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] +CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", "library", ] class Local: @@ -118,6 +118,7 @@ class Local: # Depending on conf_path self.files_path = os.path.join(self.conf_path, "files") self.global_explorer_path = os.path.join(self.conf_path, "explorer") + self.global_library_path = os.path.join(self.conf_path, "library") self.manifest_path = os.path.join(self.conf_path, "manifest") self.initial_manifest = (self.custom_initial_manifest or os.path.join(self.manifest_path, "init")) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index af9e70cb..597780d0 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -89,6 +89,7 @@ class Remote: self.type_path = os.path.join(self.conf_path, "type") self.global_explorer_path = os.path.join(self.conf_path, "explorer") + self.global_library_path = os.path.join(self.conf_path, "library") self._open_logger() diff --git a/docs/src/cdist-library.rst b/docs/src/cdist-library.rst new file mode 100644 index 00000000..ea1c95c9 --- /dev/null +++ b/docs/src/cdist-library.rst @@ -0,0 +1,61 @@ +Library +======= + +Description +----------- +The library features the sharing of common code. There is a global and +a type library to match the specific use case. + +The library feature is simply making the path to the library folder +available to all scripts. With it, they can load files out of the library +from different locations. Through it, code must only written once, which +makes code easier and less redundant. Also, better code can be provided +by shared libraries. + +Synopsis +-------- + +From `manifest`, `gencode-*` and `code-local`: + +.. code-block:: + + $__library (global library directory path) + $__type/library (type library folder) + +From the remote side (`explorer` and `code-remote`): + +.. code-block:: + + $__library (remote copied library directory path) + $__type_library (remote copied type library folder) + +*Type libraries are only available if the code is available in an object- +realated content. As example, there are not available in global explorer +or the initial manifest.* + +How to use +---------- +Because the library features only distribute files, it's not bond to +`posix shell scripts` or any programming or scripting language. To use +the library, you only need to load the library from the given path. + +In shell scripts, it works as follow (here as a `type manifest`): + +.. code-block:: sh + + # loading a global library + . "$__library/manifest" + + # loading type library + . "$__type/library/foo-bar" + + + # now, sourced shell functions can be used + # ... + +Remote handling +--------------- +The libraries may also required on the remote side. For this, the +library is completly copied to the remote. Therefor, it's completly +available on the remote side and can be used in the explorers and +generated remote code. diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index c0ac2c3e..1c001893 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -253,6 +253,11 @@ __global Directory that contains generic output like explorer. Available for: initial manifest, type manifest, type gencode, shell. +__library + Directory that contains common code. + + Available for: initial manifest, type manifest, type gencode, code scripts, + global explorer, type explorer. __messages_in File to read messages from. @@ -301,11 +306,15 @@ __target_host_tags __type Path to the current type. - Available for: type manifest, type gencode. + Available for: type manifest, type gencode, local code. __type_explorer Directory that contains the type explorers. Available for: type explorer. +__type_library + Directory that contains type-specific common code. + + Available for: type explorer, code-remote. Environment variables (for writing) ----------------------------------- diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 582c0938..a0fedd4b 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -106,6 +106,7 @@ A type consists of - explorer (optional) - gencode (optional) - nonparallel (optional) +- library (optional) Types are stored below cdist/conf/type/. Their name should always be prefixed with two underscores (__) to prevent collisions with other executables in $PATH. diff --git a/docs/src/index.rst b/docs/src/index.rst index 369d5309..d1883988 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -30,6 +30,7 @@ It natively supports IPv6 since the first release. cdist-type cdist-types cdist-explorer + cdist-library cdist-messaging cdist-parallelization cdist-inventory