ungleich-public/cdist
ungleich-public
/
cdist
7
3
Fork 11
Code Issues 57 Pull Requests 15 Projects Releases Wiki Activity

Merge branch 'feature/lib' into 'master' 

WIP: Library implementation

See merge request ungleich-public/cdist!920
This commit is contained in:
matze 2021-11-04 00:16:46 +01:00
parent e2500248f2 af1dfdebda
commit 07f8dd3b19
11 changed files with 246 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
cdist/conf/library/manifest Normal file
View File

@ -0,0 +1,103 @@
# vi: set filetype=sh:
# $__library/manifest
#
# 2020 Matthias Stecher (matthiasstecher at gmx.de)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
# This library contains:
# common helper functions helping writing the manifest
# check if type is a singleton
#
# Arguments:
# 1: name of the type, e.g. `__some_type`
is_singleton() {
if [ -f "$__global/conf/type/$1/singleton" ]; then
return 0
else
return 1
fi
}
# print object id
#
# Arguments:
# 1: type name, e.g. `__some_type`
# 2: object id; will be ignored if type is a singleton
get_object_id() {
if is_singleton "$1"; then
printf "%s" "$1"
else
printf "%s/%s" "$1" "$2"
fi
}
# Helper function to get the correct chain name. It will shorten any
# invalid character.
#
# Arguments:
# 1: the dependency chain name
_rec_normalize_chain_name() {
# spaces will be underscores
# anything other than a-zA-Z0-9 will be removed
printf "$1" | sed 's/\([[:space:]]\|-\)/_/g; s/[^a-zA-Z0-9_]//g'
}
# Recording dependencies
#
# Arguments:
# 1: the dependency chain - only chars, numbers and underscores allowed
# 2..n: the type command; no more nesting possible
rec_requires() {
# make it ready to be passed to eval
_rec_chain="$(_rec_normalize_chain_name "$1")"
eval "_rec_ch_${_rec_chain}=\"\${_rec_ch_${_rec_chain}:-} $(get_object_id "$2" "$3")\""
# execute type
shift
"$@"
}
# Clearing dependency chains
#
# Arguments:
# 1: the dependency chain name
rec_clear() {
_rec_chain="$(printf "$1" | sed 's/\([[:space:]]\|-\)/_/g; s/[^a-zA-Z0-9_]//g')"
eval "unset -v _rec_ch_${_rec_chain}"
}
# Export dependencies to current command by appending to the current environment
#
# Arguments:
# 1: the dependency chain
# 2..n: the type command; no more nesting possible
depend() {
# assemble dependency
_rec_chain="$(_rec_normalize_chain_name "$1")"
_rec_deps="$(eval "printf \"%s\" \"\${_rec_ch_${_rec_chain}:-}\"")"
# execute type
shift
require="$require $_rec_deps" "$@"
}

26
cdist/core/cdist_type.py
View File

@ -56,8 +56,10 @@ class CdistType:
self.absolute_path = os.path.join(self.base_path, self.path)
if not os.path.isdir(self.absolute_path):
raise InvalidTypeError(self.name, self.path, self.absolute_path)
self.manifest_path = os.path.join(self.name, "manifest")
self.explorer_path = os.path.join(self.name, "explorer")
self.library_path = os.path.join(self.name, "library")
self.gencode_local_path = os.path.join(self.name, "gencode-local")
self.gencode_remote_path = os.path.join(self.name, "gencode-remote")
self.manifest_path = os.path.join(self.name, "manifest")
@ -71,6 +73,8 @@ class CdistType:
self.__parameter_defaults = None
self.__deprecated_parameters = None
self._transferred_type_library = False
def __hash__(self):
return hash(self.name)
@ -296,3 +300,25 @@ class CdistType:
finally:
self.__deprecated_parameters = deprecated
return self.__deprecated_parameters
# transfer of the library required by type explorers and code-remote
def transfer_type_library(self, target):
"""Transfer the type library for the given type to the remote side.
target: Code | Explorer | Manifest
(must contain .local and .remote properties)
"""
if not self._transferred_type_library:
source = os.path.join(target.local.type_path,
self.library_path)
destination = os.path.join(target.remote.type_path,
self.library_path)
if os.path.isdir(source) and os.listdir(source):
target.remote.transfer(source, destination)
target.remote.run(["chmod", "0700", "%s/*" % (destination)])
else:
# at least, the directory should exist
target.remote.mkdir(destination)
self._transferred_type_library = True

21
cdist/core/code.py
View File

@ -39,6 +39,8 @@ common:
__cdist_type_base_path: full qualified path to the directory where
types are defined for use in type emulator
== local.type_path
__library: full qualified path to the library folder containing
common code == local.global_library_path
gencode-local
script: full qualified path to a types gencode-local
@ -107,6 +109,7 @@ class Code:
'__target_fqdn': self.target_host[2],
'__global': self.local.base_path,
'__files': self.local.files_path,
'__library': self.local.global_library_path,
'__target_host_tags': self.local.target_host_tags,
'__cdist_log_level': util.log_level_env_var_val(local.log),
'__cdist_log_level_name': util.log_level_name_env_var_val(
@ -157,7 +160,12 @@ class Code:
def transfer_code_remote(self, cdist_object):
"""Transfer the code_remote script for the given object to the
remote side."""
remote side. Will transfer the type library too if not already
copied to the remote side cause of type explorers."""
# $__type/library (if not already done)
cdist_object.cdist_type.transfer_type_library(self)
# $__object/code-remote
source = os.path.join(self.local.object_path,
cdist_object.code_remote_path)
destination = os.path.join(self.remote.object_path,
@ -188,6 +196,7 @@ class Code:
env = os.environ.copy()
env.update(self.env)
env.update({
'__type': cdist_object.cdist_type.absolute_path,
'__object': cdist_object.absolute_path,
'__object_id': cdist_object.object_id,
})
@ -196,11 +205,17 @@ class Code:
def run_code_remote(self, cdist_object):
"""Run the code-remote script for the given cdist object on the
remote side."""
# Put some env vars, to allow read only access to the parameters
# over $__object which is already on the remote side
env = {
# Put some env vars, to allow read only access to the parameters
# over $__object which is already on the remote side
'__object': os.path.join(self.remote.object_path,
cdist_object.path),
'__object_id': cdist_object.object_id,
# The libraries should be over there, too
'__library': self.remote.global_library_path,
'__type_library': os.path.join(
self.remote.type_path,
cdist_object.cdist_type.library_path),
}
return self._run_code(cdist_object, 'remote', env=env)

22
cdist/core/explorer.py
View File

@ -36,6 +36,9 @@ common:
__explorer: full qualified path to other global explorers on
remote side
-> remote.global_explorer_path
__library: full qualified path to global library files on the
remote side
-> remote.global_library_path
a global explorer is:
- a script
@ -57,6 +60,7 @@ type explorer is:
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type_explorer: full qualified path to the other type explorers on
remote side
__type_library: full qualified path to the type library files on remote
creates: nothing, returns output
@ -79,6 +83,7 @@ class Explorer:
'__target_hostname': self.target_host[1],
'__target_fqdn': self.target_host[2],
'__explorer': self.remote.global_explorer_path,
'__library': self.remote.global_library_path,
'__target_host_tags': self.local.target_host_tags,
'__cdist_log_level': util.log_level_env_var_val(self.log),
'__cdist_log_level_name': util.log_level_name_env_var_val(
@ -106,6 +111,7 @@ class Explorer:
"""
self.log.verbose("Running global explorers")
self.transfer_global_library()
self.transfer_global_explorers()
if self.jobs is None:
self._run_global_explorers_seq(out_path)
@ -163,6 +169,14 @@ class Explorer:
self.remote.run(["chmod", "0700", "{}/*".format(
self.remote.global_explorer_path)])
def transfer_global_library(self):
"""Transfer the global library files to the remote side."""
self.remote.transfer(self.local.global_library_path,
self.remote.global_library_path,
self.jobs)
self.remote.run(["chmod", "0700",
"%s/*" % (self.remote.global_library_path)])
def run_global_explorer(self, explorer):
"""Run the given global explorer and return it's output."""
script = os.path.join(self.remote.global_explorer_path, explorer)
@ -223,7 +237,9 @@ class Explorer:
'__object_name': cdist_object.name,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path,
cdist_type.explorer_path)
cdist_type.explorer_path),
'__type_library': os.path.join(self.remote.type_path,
cdist_type.library_path),
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path,
explorer)
@ -237,6 +253,10 @@ class Explorer:
self.log.trace("Skipping retransfer of type explorers for: %s",
cdist_type)
else:
# transfer library by common code
cdist_type.transfer_type_library(self)
# transfer type explorers
source = os.path.join(self.local.type_path,
cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path,

2
cdist/core/manifest.py
View File

@ -43,6 +43,7 @@ common:
types are defined for use in type emulator
== local.type_path
__files: full qualified path to the files dir
__library: full qualitfied path to the library dir
__target_host_tags: comma spearated list of host tags
initial manifest is:
@ -114,6 +115,7 @@ class Manifest:
'__target_hostname': self.target_host[1],
'__target_fqdn': self.target_host[2],
'__files': self.local.files_path,
'__library': self.local.global_library_path,
'__target_host_tags': self.local.target_host_tags,
'__cdist_log_level': util.log_level_env_var_val(self.log),
'__cdist_log_level_name': util.log_level_name_env_var_val(

3
cdist/exec/local.py
View File

@ -36,7 +36,7 @@ import cdist.message
from cdist import core
import cdist.exec.util as util
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ]
CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", "library", ]
class Local:
@ -118,6 +118,7 @@ class Local:
# Depending on conf_path
self.files_path = os.path.join(self.conf_path, "files")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.global_library_path = os.path.join(self.conf_path, "library")
self.manifest_path = os.path.join(self.conf_path, "manifest")
self.initial_manifest = (self.custom_initial_manifest or
os.path.join(self.manifest_path, "init"))

1
cdist/exec/remote.py
View File

@ -89,6 +89,7 @@ class Remote:
self.type_path = os.path.join(self.conf_path, "type")
self.global_explorer_path = os.path.join(self.conf_path, "explorer")
self.global_library_path = os.path.join(self.conf_path, "library")
self._open_logger()

61
docs/src/cdist-library.rst Normal file
View File

@ -0,0 +1,61 @@
Library
=======
Description
-----------
The library features the sharing of common code. There is a global and
a type library to match the specific use case.
The library feature is simply making the path to the library folder
available to all scripts. With it, they can load files out of the library
from different locations. Through it, code must only written once, which
makes code easier and less redundant. Also, better code can be provided
by shared libraries.
Synopsis
--------
From `manifest`, `gencode-*` and `code-local`:
.. code-block::
$__library (global library directory path)
$__type/library (type library folder)
From the remote side (`explorer` and `code-remote`):
.. code-block::
$__library (remote copied library directory path)
$__type_library (remote copied type library folder)
*Type libraries are only available if the code is available in an object-
realated content. As example, there are not available in global explorer
or the initial manifest.*
How to use
----------
Because the library features only distribute files, it's not bond to
`posix shell scripts` or any programming or scripting language. To use
the library, you only need to load the library from the given path.
In shell scripts, it works as follow (here as a `type manifest`):
.. code-block:: sh
# loading a global library
. "$__library/manifest"
# loading type library
. "$__type/library/foo-bar"
# now, sourced shell functions can be used
# ...
Remote handling
---------------
The libraries may also required on the remote side. For this, the
library is completly copied to the remote. Therefor, it's completly
available on the remote side and can be used in the explorers and
generated remote code.

11
docs/src/cdist-reference.rst.sh
View File

@ -253,6 +253,11 @@ __global
Directory that contains generic output like explorer.
Available for: initial manifest, type manifest, type gencode, shell.
__library
Directory that contains common code.
Available for: initial manifest, type manifest, type gencode, code scripts,
global explorer, type explorer.
__messages_in
File to read messages from.
@ -301,11 +306,15 @@ __target_host_tags
__type
Path to the current type.
Available for: type manifest, type gencode.
Available for: type manifest, type gencode, local code.
__type_explorer
Directory that contains the type explorers.
Available for: type explorer.
__type_library
Directory that contains type-specific common code.
Available for: type explorer, code-remote.
Environment variables (for writing)
-----------------------------------

1
docs/src/cdist-type.rst
View File

@ -106,6 +106,7 @@ A type consists of
- explorer (optional)
- gencode (optional)
- nonparallel (optional)
- library (optional)
Types are stored below cdist/conf/type/. Their name should always be prefixed with
two underscores (__) to prevent collisions with other executables in $PATH.

1
docs/src/index.rst
View File

@ -30,6 +30,7 @@ It natively supports IPv6 since the first release.
cdist-type
cdist-types
cdist-explorer
cdist-library
cdist-messaging
cdist-parallelization
cdist-inventory