This commit is contained in:
Darko Poljak 2016-07-05 20:44:24 +02:00
commit 64efa04599
29 changed files with 715 additions and 409 deletions

View file

@ -20,10 +20,10 @@
#
#
from cdist.core.cdist_type import CdistType
from cdist.core.cdist_type import NoSuchTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code
from cdist.core.cdist_type import CdistType
from cdist.core.cdist_type import NoSuchTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code

View file

@ -32,6 +32,7 @@ from cdist.util import fsproperty
log = logging.getLogger(__name__)
class IllegalObjectIdError(cdist.Error):
def __init__(self, object_id, message=None):
self.object_id = object_id
@ -40,14 +41,17 @@ class IllegalObjectIdError(cdist.Error):
def __str__(self):
return '%s: %s' % (self.message, self.object_id)
class MissingObjectIdError(cdist.Error):
def __init__(self, type_name):
self.type_name = type_name
self.message = "Type %s requires object id (is not a singleton type)" % self.type_name
self.message = ("Type %s requires object id (is not a "
"singleton type)") % self.type_name
def __str__(self):
return '%s' % (self.message)
class CdistObject(object):
"""Represents a cdist object.
@ -64,7 +68,7 @@ class CdistObject(object):
STATE_DONE = "done"
def __init__(self, cdist_type, base_path, object_marker, object_id):
self.cdist_type = cdist_type # instance of Type
self.cdist_type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
@ -74,7 +78,8 @@ class CdistObject(object):
self.sanitise_object_id()
self.name = self.join_name(self.cdist_type.name, self.object_id)
self.path = os.path.join(self.cdist_type.path, self.object_id, self.object_marker)
self.path = os.path.join(self.cdist_type.path, self.object_id,
self.object_marker)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
@ -84,12 +89,13 @@ class CdistObject(object):
@classmethod
def list_objects(cls, object_base_path, type_base_path, object_marker):
"""Return a list of object instances"""
for object_name in cls.list_object_names(object_base_path, object_marker):
for object_name in cls.list_object_names(
object_base_path, object_marker):
type_name, object_id = cls.split_name(object_name)
yield cls(cdist.core.CdistType(type_base_path, type_name),
base_path=object_base_path,
object_marker=object_marker,
object_id=object_id)
base_path=object_base_path,
object_marker=object_marker,
object_id=object_id)
@classmethod
def list_object_names(cls, object_base_path, object_marker):
@ -125,26 +131,34 @@ class CdistObject(object):
def validate_object_id(self):
if self.cdist_type.is_singleton and self.object_id:
raise IllegalObjectIdError('singleton objects can\'t have a object_id')
raise IllegalObjectIdError(('singleton objects can\'t have an '
'object_id'))
"""Validate the given object_id and raise IllegalObjectIdError if it's not valid.
"""Validate the given object_id and raise IllegalObjectIdError
if it's not valid.
"""
if self.object_id:
if self.object_marker in self.object_id.split(os.sep):
raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % self.object_marker)
raise IllegalObjectIdError(
self.object_id, ('object_id may not contain '
'\'%s\'') % self.object_marker)
if '//' in self.object_id:
raise IllegalObjectIdError(self.object_id, 'object_id may not contain //')
raise IllegalObjectIdError(
self.object_id, 'object_id may not contain //')
if self.object_id == '.':
raise IllegalObjectIdError(self.object_id, 'object_id may not be a .')
raise IllegalObjectIdError(
self.object_id, 'object_id may not be a .')
# If no object_id and type is not singleton => error out
if not self.object_id and not self.cdist_type.is_singleton:
raise MissingObjectIdError(self.cdist_type.name)
# Does not work: AttributeError: 'CdistObject' object has no attribute 'parameter_path'
# Does not work:
# AttributeError:
# 'CdistObject' object has no attribute 'parameter_path'
#"Type %s is not a singleton type - missing object id (parameters: %s)" %
# (self.cdist_type.name, self.parameters))
# "Type %s is not a singleton type - missing object id
# (parameters: %s)" % (self.cdist_type.name, self.parameters))
def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name.
@ -152,7 +166,8 @@ class CdistObject(object):
Mainly intended to create objects when resolving requirements.
e.g:
<CdistObject __foo/bar>.object_from_name('__other/object') -> <CdistObject __other/object>
<CdistObject __foo/bar>.object_from_name('__other/object') ->
<CdistObject __other/object>
"""
@ -164,7 +179,8 @@ class CdistObject(object):
cdist_type = self.cdist_type.__class__(type_path, type_name)
return self.__class__(cdist_type, base_path, object_marker, object_id=object_id)
return self.__class__(cdist_type, base_path, object_marker,
object_id=object_id)
def __repr__(self):
return '<CdistObject %s>' % self.name
@ -172,7 +188,7 @@ class CdistObject(object):
def __eq__(self, other):
"""define equality as 'name is the same'"""
return self.name == other.name
def __hash__(self):
return hash(self.name)
@ -205,14 +221,22 @@ class CdistObject(object):
# return relative path
return os.path.join(self.path, "explorer")
requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require'))
autorequire = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path))
explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path))
state = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.absolute_path, "state"))
source = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, "source"))
code_local = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_local_path))
code_remote = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
requirements = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, 'require'))
autorequire = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
parameters = fsproperty.DirectoryDictProperty(
lambda obj: os.path.join(obj.base_path, obj.parameter_path))
explorers = fsproperty.DirectoryDictProperty(
lambda obj: os.path.join(obj.base_path, obj.explorer_path))
state = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.absolute_path, "state"))
source = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, "source"))
code_local = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.base_path, obj.code_local_path))
code_remote = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
@property
def exists(self):
@ -224,10 +248,12 @@ class CdistObject(object):
"""
try:
os.makedirs(self.absolute_path, exist_ok=allow_overwrite)
absolute_parameter_path = os.path.join(self.base_path, self.parameter_path)
absolute_parameter_path = os.path.join(self.base_path,
self.parameter_path)
os.makedirs(absolute_parameter_path, exist_ok=allow_overwrite)
except EnvironmentError as error:
raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error))
raise cdist.Error(('Error creating directories for cdist object: '
'%s: %s') % (self, error))
def requirements_unfinished(self, requirements):
"""Return state whether requirements are satisfied"""

View file

@ -24,6 +24,7 @@ import os
import cdist
class NoSuchTypeError(cdist.Error):
def __init__(self, name, type_path, type_absolute_path):
self.name = name
@ -31,7 +32,8 @@ class NoSuchTypeError(cdist.Error):
self.type_absolute_path = type_absolute_path
def __str__(self):
return "Type '%s' does not exist at %s" % (self.type_path, self.type_absolute_path)
return "Type '%s' does not exist at %s" % (
self.type_path, self.type_absolute_path)
class CdistType(object):
@ -75,13 +77,13 @@ class CdistType(object):
"""Return a list of type names"""
return os.listdir(base_path)
_instances = {}
def __new__(cls, *args, **kwargs):
"""only one instance of each named type may exist"""
# name is second argument
name = args[1]
if not name in cls._instances:
if name not in cls._instances:
instance = super(CdistType, cls).__new__(cls)
cls._instances[name] = instance
# return instance so __init__ is called
@ -103,7 +105,8 @@ class CdistType(object):
@property
def is_install(self):
"""Check whether a type is used for installation (if not: for configuration)"""
"""Check whether a type is used for installation
(if not: for configuration)"""
return os.path.isfile(os.path.join(self.absolute_path, "install"))
@property
@ -111,7 +114,8 @@ class CdistType(object):
"""Return a list of available explorers"""
if not self.__explorers:
try:
self.__explorers = os.listdir(os.path.join(self.absolute_path, "explorer"))
self.__explorers = os.listdir(os.path.join(self.absolute_path,
"explorer"))
except EnvironmentError:
# error ignored
self.__explorers = []
@ -123,7 +127,9 @@ class CdistType(object):
if not self.__required_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "required")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"required")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -139,7 +145,9 @@ class CdistType(object):
if not self.__required_multiple_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "required_multiple")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"required_multiple")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -155,7 +163,9 @@ class CdistType(object):
if not self.__optional_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "optional")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"optional")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -171,7 +181,9 @@ class CdistType(object):
if not self.__optional_multiple_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "optional_multiple")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"optional_multiple")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -187,7 +199,9 @@ class CdistType(object):
if not self.__boolean_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "boolean")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"boolean")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -202,7 +216,9 @@ class CdistType(object):
if not self.__parameter_defaults:
defaults = {}
try:
defaults_dir = os.path.join(self.absolute_path, "parameter", "default")
defaults_dir = os.path.join(self.absolute_path,
"parameter",
"default")
for name in os.listdir(defaults_dir):
try:
with open(os.path.join(defaults_dir, name)) as fd:

View file

@ -37,15 +37,17 @@ common:
PATH: prepend directory with type emulator symlinks == local.bin_path
__target_host: the target host we are working on
__cdist_manifest: full qualified path of the manifest == script
__cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator
== local.type_path
__cdist_type_base_path: full qualified path to the directory where
types are defined for use in type emulator
== local.type_path
gencode-local
script: full qualified path to a types gencode-local
env:
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__global: full qualified path to the global
output dir == local.out_path
__object: full qualified path to the object's dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
@ -59,7 +61,8 @@ gencode-remote
env:
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__global: full qualified path to the global
output dir == local.out_path
__object: full qualified path to the object's dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
@ -98,7 +101,8 @@ class Code(object):
def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.cdist_type
script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which))
script = os.path.join(self.local.type_path,
getattr(cdist_type, 'gencode_%s_path' % which))
if os.path.isfile(script):
env = os.environ.copy()
env.update(self.env)
@ -108,8 +112,9 @@ class Code(object):
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
})
message_prefix=cdist_object.name
return self.local.run_script(script, env=env, return_output=True, message_prefix=message_prefix)
message_prefix = cdist_object.name
return self.local.run_script(script, env=env, return_output=True,
message_prefix=message_prefix)
def run_gencode_local(self, cdist_object):
"""Run the gencode-local script for the given cdist object."""
@ -120,21 +125,26 @@ class Code(object):
return self._run_gencode(cdist_object, 'remote')
def transfer_code_remote(self, cdist_object):
"""Transfer the code_remote script for the given object to the remote side."""
source = os.path.join(self.local.object_path, cdist_object.code_remote_path)
destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path)
"""Transfer the code_remote script for the given object to the
remote side."""
source = os.path.join(self.local.object_path,
cdist_object.code_remote_path)
destination = os.path.join(self.remote.object_path,
cdist_object.code_remote_path)
# FIXME: BUG: do not create destination, but top level of destination!
self.remote.mkdir(destination)
self.remote.transfer(source, destination)
def _run_code(self, cdist_object, which, env=None):
which_exec = getattr(self, which)
script = os.path.join(which_exec.object_path, getattr(cdist_object, 'code_%s_path' % which))
script = os.path.join(which_exec.object_path,
getattr(cdist_object, 'code_%s_path' % which))
return which_exec.run_script(script, env=env)
def run_code_local(self, cdist_object):
"""Run the code-local script for the given cdist object."""
# Put some env vars, to allow read only access to the parameters over $__object
# Put some env vars, to allow read only access to the parameters
# over $__object
env = os.environ.copy()
env.update(self.env)
env.update({
@ -144,10 +154,13 @@ class Code(object):
return self._run_code(cdist_object, 'local', env=env)
def run_code_remote(self, cdist_object):
"""Run the code-remote script for the given cdist object on the remote side."""
# Put some env vars, to allow read only access to the parameters over $__object which is already on the remote side
"""Run the code-remote script for the given cdist object on the
remote side."""
# Put some env vars, to allow read only access to the parameters
# over $__object which is already on the remote side
env = {
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object': os.path.join(self.remote.object_path,
cdist_object.path),
'__object_id': cdist_object.object_id,
}
return self._run_code(cdist_object, 'remote', env=env)

View file

@ -31,7 +31,8 @@ common:
runs only remotely, needs local and remote to construct paths
env:
__explorer: full qualified path to other global explorers on remote side
__explorer: full qualified path to other global explorers on
remote side
-> remote.global_explorer_path
a global explorer is:
@ -52,7 +53,8 @@ type explorer is:
__object: full qualified path to the object's remote dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type_explorer: full qualified path to the other type explorers on remote side
__type_explorer: full qualified path to the other type explorers on
remote side
creates: nothing, returns output
@ -76,7 +78,7 @@ class Explorer(object):
}
self._type_explorers_transferred = []
### global
# global
def list_global_explorer_names(self):
"""Return a list of global explorer names."""
@ -98,15 +100,17 @@ class Explorer(object):
def transfer_global_explorers(self):
"""Transfer the global explorers to the remote side."""
self.remote.mkdir(self.remote.global_explorer_path)
self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path)
self.remote.run(["chmod", "0700", "%s/*" % (self.remote.global_explorer_path)])
self.remote.transfer(self.local.global_explorer_path,
self.remote.global_explorer_path)
self.remote.run(["chmod", "0700",
"%s/*" % (self.remote.global_explorer_path)])
def run_global_explorer(self, explorer):
"""Run the given global explorer and return it's output."""
script = os.path.join(self.remote.global_explorer_path, explorer)
return self.remote.run_script(script, env=self.env, return_output=True)
### type
# type
def list_type_explorer_names(self, cdist_type):
"""Return a list of explorer names for the given type."""
@ -121,37 +125,48 @@ class Explorer(object):
in the object.
"""
self.log.debug("Transfering type explorers for type: %s", cdist_object.cdist_type)
self.log.debug("Transfering type explorers for type: %s",
cdist_object.cdist_type)
self.transfer_type_explorers(cdist_object.cdist_type)
self.log.debug("Transfering object parameters for object: %s", cdist_object.name)
self.log.debug("Transfering object parameters for object: %s",
cdist_object.name)
self.transfer_object_parameters(cdist_object)
for explorer in self.list_type_explorer_names(cdist_object.cdist_type):
output = self.run_type_explorer(explorer, cdist_object)
self.log.debug("Running type explorer '%s' for object '%s'", explorer, cdist_object.name)
self.log.debug("Running type explorer '%s' for object '%s'",
explorer, cdist_object.name)
cdist_object.explorers[explorer] = output
def run_type_explorer(self, explorer, cdist_object):
"""Run the given type explorer for the given object and return it's output."""
"""Run the given type explorer for the given object and return
it's output."""
cdist_type = cdist_object.cdist_type
env = self.env.copy()
env.update({
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object': os.path.join(self.remote.object_path,
cdist_object.path),
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path)
'__type_explorer': os.path.join(self.remote.type_path,
cdist_type.explorer_path)
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer)
script = os.path.join(self.remote.type_path, cdist_type.explorer_path,
explorer)
return self.remote.run_script(script, env=env, return_output=True)
def transfer_type_explorers(self, cdist_type):
"""Transfer the type explorers for the given type to the remote side."""
"""Transfer the type explorers for the given type to the
remote side."""
if cdist_type.explorers:
if cdist_type.name in self._type_explorers_transferred:
self.log.debug("Skipping retransfer of type explorers for: %s", cdist_type)
self.log.debug("Skipping retransfer of type explorers for: %s",
cdist_type)
else:
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
source = os.path.join(self.local.type_path,
cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path,
cdist_type.explorer_path)
self.remote.mkdir(destination)
self.remote.transfer(source, destination)
self.remote.run(["chmod", "0700", "%s/*" % (destination)])
@ -160,7 +175,9 @@ class Explorer(object):
def transfer_object_parameters(self, cdist_object):
"""Transfer the parameters for the given object to the remote side."""
if cdist_object.parameters:
source = os.path.join(self.local.object_path, cdist_object.parameter_path)
destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
source = os.path.join(self.local.object_path,
cdist_object.parameter_path)
destination = os.path.join(self.remote.object_path,
cdist_object.parameter_path)
self.remote.mkdir(destination)
self.remote.transfer(source, destination)

View file

@ -32,9 +32,11 @@ common:
env:
PATH: prepend directory with type emulator symlinks == local.bin_path
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__global: full qualified path to the global
output dir == local.out_path
__cdist_manifest: full qualified path of the manifest == script
__cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator
__cdist_type_base_path: full qualified path to the directory where
types are defined for use in type emulator
== local.type_path
__files: full qualified path to the files dir
@ -58,6 +60,7 @@ type manifeste is:
creates: new objects through type emulator
'''
class NoInitialManifestError(cdist.Error):
"""
Display missing initial manifest:
@ -72,7 +75,9 @@ class NoInitialManifestError(cdist.Error):
if user_supplied:
if os.path.islink(manifest_path):
self.message = "%s: %s -> %s" % (msg_header, manifest_path, os.path.realpath(manifest_path))
self.message = "%s: %s -> %s" % (
msg_header, manifest_path,
os.path.realpath(manifest_path))
else:
self.message = "%s: %s" % (msg_header, manifest_path)
else:
@ -94,14 +99,15 @@ class Manifest(object):
self.env = {
'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
'__cdist_type_base_path': self.local.type_path, # for use in type emulator
# for use in type emulator
'__cdist_type_base_path': self.local.type_path,
'__global': self.local.base_path,
'__target_host': self.target_host,
'__files': self.local.files_path,
}
if self.log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__cdist_debug': "yes" })
if self.log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__cdist_debug': "yes"})
def env_initial_manifest(self, initial_manifest):
env = os.environ.copy()
@ -124,11 +130,14 @@ class Manifest(object):
if not os.path.isfile(initial_manifest):
raise NoInitialManifestError(initial_manifest, user_supplied)
message_prefix="initialmanifest"
self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix)
message_prefix = "initialmanifest"
self.local.run_script(initial_manifest,
env=self.env_initial_manifest(initial_manifest),
message_prefix=message_prefix)
def env_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
type_manifest = os.path.join(self.local.type_path,
cdist_object.cdist_type.manifest_path)
env = os.environ.copy()
env.update(self.env)
env.update({
@ -143,7 +152,10 @@ class Manifest(object):
return env
def run_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
type_manifest = os.path.join(self.local.type_path,
cdist_object.cdist_type.manifest_path)
message_prefix = cdist_object.name
if os.path.isfile(type_manifest):
self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix)
self.local.run_script(type_manifest,
env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix)