From 7f95bbd535da3965f1a2209b643bdb34cfc26174 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 23 Jul 2020 21:41:47 +0200 Subject: [PATCH 1/7] Add global library environment variables to local script executions Threads the `conf/library` folder as a further configuration folder and exports it via the `$__library` environment varibale to the manifest and gencode-* scripts. With it, shell scripts can be sourced as library. For types, `$__type/library` must be used instead to reach these scripts. --- cdist/core/code.py | 3 +++ cdist/core/manifest.py | 2 ++ cdist/exec/local.py | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 1e9b4f80..019300c5 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -39,6 +39,8 @@ common: __cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator == local.type_path + __library: full qualified path to the library folder containing + common code == local.library_path gencode-local script: full qualified path to a types gencode-local @@ -107,6 +109,7 @@ class Code: '__target_fqdn': self.target_host[2], '__global': self.local.base_path, '__files': self.local.files_path, + '__library': self.local.library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(local.log), '__cdist_log_level_name': util.log_level_name_env_var_val( diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 390340d4..382a38e2 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -43,6 +43,7 @@ common: types are defined for use in type emulator == local.type_path __files: full qualified path to the files dir + __library: full qualitfied path to the library dir __target_host_tags: comma spearated list of host tags initial manifest is: @@ -115,6 +116,7 @@ class Manifest: '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__files': self.local.files_path, + '__library': self.local.library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( diff --git a/cdist/exec/local.py b/cdist/exec/local.py index e0aab190..482a32be 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -36,7 +36,7 @@ import cdist.message from cdist import core import cdist.exec.util as util -CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] +CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", "library", ] class Local: @@ -123,6 +123,7 @@ class Local: os.path.join(self.manifest_path, "init")) self.type_path = os.path.join(self.conf_path, "type") + self.library_path = os.path.join(self.conf_path, "library") def _init_object_marker(self): self.object_marker_file = os.path.join(self.base_path, "object_marker") From e779289e5d72cb0efc1bd9fda133b0e2617fb742 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sat, 25 Jul 2020 12:40:49 +0200 Subject: [PATCH 2/7] Added library support for remote explorer execution. The library is now copied to the remote side if any explorer will executed. This allows to share code across multiple explorers. While the library code is quite different for different use cases (e.g. manifest, gencode, remote side), everything get copied. This may be a waste and could be categorized better. It would result in only copy the general and required folder to the remote. --- cdist/core/cdist_type.py | 1 + cdist/core/code.py | 2 +- cdist/core/explorer.py | 31 ++++++++++++++++++++++++++++++- cdist/core/manifest.py | 2 +- cdist/exec/local.py | 2 +- cdist/exec/remote.py | 1 + 6 files changed, 35 insertions(+), 4 deletions(-) diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index c0329c8a..044c2b9d 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -58,6 +58,7 @@ class CdistType: raise InvalidTypeError(self.name, self.path, self.absolute_path) self.manifest_path = os.path.join(self.name, "manifest") self.explorer_path = os.path.join(self.name, "explorer") + self.library_path = os.path.join(self.name, "library") self.gencode_local_path = os.path.join(self.name, "gencode-local") self.gencode_remote_path = os.path.join(self.name, "gencode-remote") self.manifest_path = os.path.join(self.name, "manifest") diff --git a/cdist/core/code.py b/cdist/core/code.py index 019300c5..1e76386d 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -109,7 +109,7 @@ class Code: '__target_fqdn': self.target_host[2], '__global': self.local.base_path, '__files': self.local.files_path, - '__library': self.local.library_path, + '__library': self.local.global_library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(local.log), '__cdist_log_level_name': util.log_level_name_env_var_val( diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index a3baa959..b3a55da6 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -36,6 +36,9 @@ common: __explorer: full qualified path to other global explorers on remote side -> remote.global_explorer_path + __library: full qualified path to global library files on the + remote side + -> remote.global_library_path a global explorer is: - a script @@ -57,6 +60,7 @@ type explorer is: __object_fq: full qualified object id, iow: $type.name + / + object_id __type_explorer: full qualified path to the other type explorers on remote side + __type_library: full qualified path to the type library files on remote creates: nothing, returns output @@ -79,6 +83,7 @@ class Explorer: '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__explorer': self.remote.global_explorer_path, + '__library': self.remote.global_library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( @@ -106,6 +111,7 @@ class Explorer: """ self.log.verbose("Running global explorers") + self.transfer_global_library() self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) @@ -164,6 +170,14 @@ class Explorer: self.remote.run(["chmod", "0700", "%s/*" % (self.remote.global_explorer_path)]) + def transfer_global_library(self): + """Transfer the global library files to the remote side.""" + self.remote.transfer(self.local.global_library_path, + self.remote.global_library_path, + self.jobs) + self.remote.run(["chmod", "0700", + "%s/*" % (self.remote.global_library_path)]) + def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" script = os.path.join(self.remote.global_explorer_path, explorer) @@ -225,7 +239,9 @@ class Explorer: '__object_name': cdist_object.name, '__object_fq': cdist_object.path, '__type_explorer': os.path.join(self.remote.type_path, - cdist_type.explorer_path) + cdist_type.explorer_path), + '__type_library': os.path.join(self.remote.type_path, + cdist_type.library_path), }) script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer) @@ -239,6 +255,8 @@ class Explorer: self.log.trace(("Skipping retransfer of type explorers " "for: %s"), cdist_type) else: + self.transfer_type_library(cdist_type) + source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, @@ -247,6 +265,17 @@ class Explorer: self.remote.run(["chmod", "0700", "%s/*" % (destination)]) self._type_explorers_transferred.append(cdist_type.name) + def transfer_type_library(self, cdist_type): + """Transfer the type library for the given type to the + remote side.""" + source = os.path.join(self.local.type_path, + cdist_type.library_path) + if os.path.exists(source) and os.listdir(source): + destination = os.path.join(self.remote.type_path, + cdist_type.library_path) + self.remote.transfer(source, destination) + self.remote.run(["chmod", "0700", "%s/*" % (destination)]) + def transfer_object_parameters(self, cdist_object): """Transfer the parameters for the given object to the remote side.""" if cdist_object.parameters: diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 382a38e2..fbb5825b 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -116,7 +116,7 @@ class Manifest: '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__files': self.local.files_path, - '__library': self.local.library_path, + '__library': self.local.global_library_path, '__target_host_tags': self.local.target_host_tags, '__cdist_log_level': util.log_level_env_var_val(self.log), '__cdist_log_level_name': util.log_level_name_env_var_val( diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 482a32be..7533d597 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -118,12 +118,12 @@ class Local: # Depending on conf_path self.files_path = os.path.join(self.conf_path, "files") self.global_explorer_path = os.path.join(self.conf_path, "explorer") + self.global_library_path = os.path.join(self.conf_path, "library") self.manifest_path = os.path.join(self.conf_path, "manifest") self.initial_manifest = (self.custom_initial_manifest or os.path.join(self.manifest_path, "init")) self.type_path = os.path.join(self.conf_path, "type") - self.library_path = os.path.join(self.conf_path, "library") def _init_object_marker(self): self.object_marker_file = os.path.join(self.base_path, "object_marker") diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index e5af2f34..0bf04313 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -91,6 +91,7 @@ class Remote: self.type_path = os.path.join(self.conf_path, "type") self.global_explorer_path = os.path.join(self.conf_path, "explorer") + self.global_library_path = os.path.join(self.conf_path, "library") self._open_logger() From 2c298c22ea93dbd7556291d16c9394d67df93900 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 26 Jul 2020 17:54:15 +0200 Subject: [PATCH 3/7] Add library access to code-remote script. The code-remote script can now access the library files, too. It gets the same environment variables `$__library` (global library) and `$__type_library` (type specific library) like the type explorers. The library files are copied to the remote host if this is not already done for the type explorers. The method for this was moved to the cdist type class to gain better generalisation and accessability for both parties. --- cdist/core/cdist_type.py | 25 +++++++++++++++++++++++++ cdist/core/code.py | 18 ++++++++++++++---- cdist/core/explorer.py | 15 +++------------ 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 044c2b9d..f44a778c 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -56,6 +56,7 @@ class CdistType: self.absolute_path = os.path.join(self.base_path, self.path) if not os.path.isdir(self.absolute_path): raise InvalidTypeError(self.name, self.path, self.absolute_path) + self.manifest_path = os.path.join(self.name, "manifest") self.explorer_path = os.path.join(self.name, "explorer") self.library_path = os.path.join(self.name, "library") @@ -72,6 +73,8 @@ class CdistType: self.__parameter_defaults = None self.__deprecated_parameters = None + self._transferred_type_library = False + def __hash__(self): return hash(self.name) @@ -297,3 +300,25 @@ class CdistType: finally: self.__deprecated_parameters = deprecated return self.__deprecated_parameters + + # transfer of the library required by type explorers and code-remote + def transfer_type_library(self, target): + """Transfer the type library for the given type to the remote side. + + target: Code | Explorer | Manifest + (must contain .local and .remote properties) + """ + if not self._transferred_type_library: + source = os.path.join(target.local.type_path, + self.library_path) + destination = os.path.join(target.remote.type_path, + self.library_path) + + if os.path.isdir(source) and os.listdir(source): + target.remote.transfer(source, destination) + target.remote.run(["chmod", "0700", "%s/*" % (destination)]) + else: + # at least, the directory should exist + target.remote.mkdir(destination) + + self._transferred_type_library = True diff --git a/cdist/core/code.py b/cdist/core/code.py index 1e76386d..85e689d0 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -40,7 +40,7 @@ common: types are defined for use in type emulator == local.type_path __library: full qualified path to the library folder containing - common code == local.library_path + common code == local.global_library_path gencode-local script: full qualified path to a types gencode-local @@ -160,7 +160,12 @@ class Code: def transfer_code_remote(self, cdist_object): """Transfer the code_remote script for the given object to the - remote side.""" + remote side. Will transfer the type library too if not already + copied to the remote side cause of type explorers.""" + # $__type/library (if not already done) + cdist_object.cdist_type.transfer_type_library(self) + + # $__object/code-remote source = os.path.join(self.local.object_path, cdist_object.code_remote_path) destination = os.path.join(self.remote.object_path, @@ -199,11 +204,16 @@ class Code: def run_code_remote(self, cdist_object): """Run the code-remote script for the given cdist object on the remote side.""" - # Put some env vars, to allow read only access to the parameters - # over $__object which is already on the remote side env = { + # Put some env vars, to allow read only access to the parameters + # over $__object which is already on the remote side '__object': os.path.join(self.remote.object_path, cdist_object.path), '__object_id': cdist_object.object_id, + + # The libraries should be over there, too + '__library': self.remote.global_library_path, + '__type_library': os.path.join(self.remote.type_path, + cdist_object.cdist_type.library_path), } return self._run_code(cdist_object, 'remote', env=env) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index b3a55da6..1d2f7b98 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -255,8 +255,10 @@ class Explorer: self.log.trace(("Skipping retransfer of type explorers " "for: %s"), cdist_type) else: - self.transfer_type_library(cdist_type) + # transfer library by common code + cdist_type.transfer_type_library(self) + # transfer type explorers source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, @@ -265,17 +267,6 @@ class Explorer: self.remote.run(["chmod", "0700", "%s/*" % (destination)]) self._type_explorers_transferred.append(cdist_type.name) - def transfer_type_library(self, cdist_type): - """Transfer the type library for the given type to the - remote side.""" - source = os.path.join(self.local.type_path, - cdist_type.library_path) - if os.path.exists(source) and os.listdir(source): - destination = os.path.join(self.remote.type_path, - cdist_type.library_path) - self.remote.transfer(source, destination) - self.remote.run(["chmod", "0700", "%s/*" % (destination)]) - def transfer_object_parameters(self, cdist_object): """Transfer the parameters for the given object to the remote side.""" if cdist_object.parameters: From 839c4440b51ef5d72125bcb47b47f836193a66ae Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Sun, 26 Jul 2020 19:35:31 +0200 Subject: [PATCH 4/7] Fix pycodestyle error E128 If fixing E128, the E501 would occur. A bigger change was required. --- cdist/core/code.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 85e689d0..05280aa5 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -213,7 +213,8 @@ class Code: # The libraries should be over there, too '__library': self.remote.global_library_path, - '__type_library': os.path.join(self.remote.type_path, - cdist_object.cdist_type.library_path), + '__type_library': os.path.join( + self.remote.type_path, + cdist_object.cdist_type.library_path), } return self._run_code(cdist_object, 'remote', env=env) From 76bc9546152e6d0026a26df49480f02a2a801f1c Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Tue, 28 Jul 2020 18:29:03 +0200 Subject: [PATCH 5/7] Added sample library script for manifest dependency handling This adds a draft of a library shell script which can be sourced to utilise all functions. Can be extended with various checks, which are not implemented yet. Original, it was done so there is a file in the global library directory that no errors occur. This should fix the unit tests. --- cdist/conf/library/manifest | 103 ++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100644 cdist/conf/library/manifest diff --git a/cdist/conf/library/manifest b/cdist/conf/library/manifest new file mode 100644 index 00000000..2592beec --- /dev/null +++ b/cdist/conf/library/manifest @@ -0,0 +1,103 @@ +# vi: set filetype=sh: +# $__library/manifest +# +# 2020 Matthias Stecher (matthiasstecher at gmx.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# This library contains: +# common helper functions helping writing the manifest + + +# check if type is a singleton +# +# Arguments: +# 1: name of the type, e.g. `__some_type` +is_singleton() { + if [ -f "$__global/conf/type/$1/singleton" ]; then + return 0 + else + return 1 + fi +} + +# print object id +# +# Arguments: +# 1: type name, e.g. `__some_type` +# 2: object id; will be ignored if type is a singleton +get_object_id() { + if is_singleton "$1"; then + printf "%s" "$1" + else + printf "%s/%s" "$1" "$2" + fi +} + + +# Helper function to get the correct chain name. It will shorten any +# invalid character. +# +# Arguments: +# 1: the dependency chain name +_rec_normalize_chain_name() { + # spaces will be underscores + # anything other than a-zA-Z0-9 will be removed + printf "$1" | sed 's/\([[:space:]]\|-\)/_/g; s/[^a-zA-Z0-9_]//g' +} + + +# Recording dependencies +# +# Arguments: +# 1: the dependency chain - only chars, numbers and underscores allowed +# 2..n: the type command; no more nesting possible +rec_requires() { + # make it ready to be passed to eval + _rec_chain="$(_rec_normalize_chain_name "$1")" + eval "_rec_ch_${_rec_chain}=\"\${_rec_ch_${_rec_chain}:-} $(get_object_id "$2" "$3")\"" + + # execute type + shift + "$@" +} + + +# Clearing dependency chains +# +# Arguments: +# 1: the dependency chain name +rec_clear() { + _rec_chain="$(printf "$1" | sed 's/\([[:space:]]\|-\)/_/g; s/[^a-zA-Z0-9_]//g')" + eval "unset -v _rec_ch_${_rec_chain}" +} + + +# Export dependencies to current command by appending to the current environment +# +# Arguments: +# 1: the dependency chain +# 2..n: the type command; no more nesting possible +depend() { + # assemble dependency + _rec_chain="$(_rec_normalize_chain_name "$1")" + _rec_deps="$(eval "printf \"%s\" \"\${_rec_ch_${_rec_chain}:-}\"")" + + # execute type + shift + require="$require $_rec_deps" "$@" +} From f2e589f2f80f8c1bf09530bb8847ab6ae3863da0 Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 30 Jul 2020 09:54:30 +0200 Subject: [PATCH 6/7] Update reference manpage and add `$__type` to code-local To access the type library from `code-local`, it is required to have the `$__type` variable. Else, the type-specific library can not be accessed. --- cdist/core/code.py | 1 + docs/src/cdist-reference.rst.sh | 11 ++++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cdist/core/code.py b/cdist/core/code.py index 05280aa5..0971072b 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -196,6 +196,7 @@ class Code: env = os.environ.copy() env.update(self.env) env.update({ + '__type': cdist_object.cdist_type.absolute_path, '__object': cdist_object.absolute_path, '__object_id': cdist_object.object_id, }) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index c0ac2c3e..1c001893 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -253,6 +253,11 @@ __global Directory that contains generic output like explorer. Available for: initial manifest, type manifest, type gencode, shell. +__library + Directory that contains common code. + + Available for: initial manifest, type manifest, type gencode, code scripts, + global explorer, type explorer. __messages_in File to read messages from. @@ -301,11 +306,15 @@ __target_host_tags __type Path to the current type. - Available for: type manifest, type gencode. + Available for: type manifest, type gencode, local code. __type_explorer Directory that contains the type explorers. Available for: type explorer. +__type_library + Directory that contains type-specific common code. + + Available for: type explorer, code-remote. Environment variables (for writing) ----------------------------------- From af1dfdebdace45212b1501a6d36dc30e1624c12e Mon Sep 17 00:00:00 2001 From: Matthias Stecher Date: Thu, 30 Jul 2020 09:56:18 +0200 Subject: [PATCH 7/7] New manpage for the library stuff Added an own manpage for libraries. Also did some things around it .. --- docs/src/cdist-library.rst | 61 ++++++++++++++++++++++++++++++++++++++ docs/src/cdist-type.rst | 1 + docs/src/index.rst | 1 + 3 files changed, 63 insertions(+) create mode 100644 docs/src/cdist-library.rst diff --git a/docs/src/cdist-library.rst b/docs/src/cdist-library.rst new file mode 100644 index 00000000..ea1c95c9 --- /dev/null +++ b/docs/src/cdist-library.rst @@ -0,0 +1,61 @@ +Library +======= + +Description +----------- +The library features the sharing of common code. There is a global and +a type library to match the specific use case. + +The library feature is simply making the path to the library folder +available to all scripts. With it, they can load files out of the library +from different locations. Through it, code must only written once, which +makes code easier and less redundant. Also, better code can be provided +by shared libraries. + +Synopsis +-------- + +From `manifest`, `gencode-*` and `code-local`: + +.. code-block:: + + $__library (global library directory path) + $__type/library (type library folder) + +From the remote side (`explorer` and `code-remote`): + +.. code-block:: + + $__library (remote copied library directory path) + $__type_library (remote copied type library folder) + +*Type libraries are only available if the code is available in an object- +realated content. As example, there are not available in global explorer +or the initial manifest.* + +How to use +---------- +Because the library features only distribute files, it's not bond to +`posix shell scripts` or any programming or scripting language. To use +the library, you only need to load the library from the given path. + +In shell scripts, it works as follow (here as a `type manifest`): + +.. code-block:: sh + + # loading a global library + . "$__library/manifest" + + # loading type library + . "$__type/library/foo-bar" + + + # now, sourced shell functions can be used + # ... + +Remote handling +--------------- +The libraries may also required on the remote side. For this, the +library is completly copied to the remote. Therefor, it's completly +available on the remote side and can be used in the explorers and +generated remote code. diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 582c0938..a0fedd4b 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -106,6 +106,7 @@ A type consists of - explorer (optional) - gencode (optional) - nonparallel (optional) +- library (optional) Types are stored below cdist/conf/type/. Their name should always be prefixed with two underscores (__) to prevent collisions with other executables in $PATH. diff --git a/docs/src/index.rst b/docs/src/index.rst index 31c044dc..a6f36582 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -30,6 +30,7 @@ It natively supports IPv6 since the first release. cdist-type cdist-types cdist-explorer + cdist-library cdist-messaging cdist-parallelization cdist-inventory