Darko Poljak 7 years ago
parent
commit
64efa04599
  1. 11
      cdist/__init__.py
  2. 121
      cdist/config.py
  3. 14
      cdist/core/__init__.py
  4. 82
      cdist/core/cdist_object.py
  5. 38
      cdist/core/cdist_type.py
  6. 43
      cdist/core/code.py
  7. 55
      cdist/core/explorer.py
  8. 34
      cdist/core/manifest.py
  9. 119
      cdist/emulator.py
  10. 46
      cdist/exec/local.py
  11. 17
      cdist/exec/remote.py
  12. 12
      cdist/exec/util.py
  13. 1
      cdist/log.py
  14. 7
      cdist/message.py
  15. 10
      cdist/shell.py
  16. 10
      cdist/sphinxext/manpage.py
  17. 4
      cdist/test/__init__.py
  18. 3
      cdist/test/banner/__init__.py
  19. 114
      cdist/test/cdist_object/__init__.py
  20. 36
      cdist/test/cdist_type/__init__.py
  21. 46
      cdist/test/code/__init__.py
  22. 41
      cdist/test/config/__init__.py
  23. 68
      cdist/test/emulator/__init__.py
  24. 51
      cdist/test/exec/local.py
  25. 49
      cdist/test/exec/remote.py
  26. 51
      cdist/test/explorer/__init__.py
  27. 20
      cdist/test/manifest/__init__.py
  28. 6
      cdist/test/message/__init__.py
  29. 15
      cdist/util/fsproperty.py

11
cdist/__init__.py vendored

@ -44,25 +44,30 @@ BANNER = """
REMOTE_COPY = "scp -o User=root"
REMOTE_EXEC = "ssh -o User=root"
class Error(Exception):
"""Base exception class for this project"""
pass
class UnresolvableRequirementsError(cdist.Error):
"""Resolving requirements failed"""
pass
class CdistObjectError(Error):
"""Something went wrong with an object"""
def __init__(self, cdist_object, message):
self.name = cdist_object.name
self.source = " ".join(cdist_object.source)
self.message = message
def __str__(self):
return '%s: %s (defined at %s)' % (self.name, self.message, self.source)
return '%s: %s (defined at %s)' % (self.name,
self.message,
self.source)
def file_to_list(filename):
"""Return list from \n seperated file"""

121
cdist/config.py vendored

@ -35,19 +35,21 @@ import cdist.exec.remote
from cdist import core
class Config(object):
"""Cdist main class to hold arbitrary data"""
def __init__(self, local, remote, dry_run=False):
self.local = local
self.remote = remote
self.log = logging.getLogger(self.local.target_host)
self.dry_run = dry_run
self.local = local
self.remote = remote
self.log = logging.getLogger(self.local.target_host)
self.dry_run = dry_run
self.explorer = core.Explorer(self.local.target_host, self.local, self.remote)
self.explorer = core.Explorer(self.local.target_host, self.local,
self.remote)
self.manifest = core.Manifest(self.local.target_host, self.local)
self.code = core.Code(self.local.target_host, self.local, self.remote)
self.code = core.Code(self.local.target_host, self.local, self.remote)
def _init_files_dirs(self):
"""Prepare files and directories for the run"""
@ -65,7 +67,7 @@ class Config(object):
try:
for host in fileinput.input(files=(source)):
# remove leading and trailing whitespace
yield host.strip()
yield host.strip()
except (IOError, OSError) as e:
raise cdist.Error("Error reading hosts from \'{}\'".format(
source))
@ -74,7 +76,6 @@ class Config(object):
for host in source:
yield host
@classmethod
def commandline(cls, args):
"""Configure remote system"""
@ -84,27 +85,29 @@ class Config(object):
log = logging.getLogger("cdist")
if args.manifest == '-' and args.hostfile == '-':
raise cdist.Error(("Cannot read both, manifest and host file, "
"from stdin"))
raise cdist.Error(("Cannot read both, manifest and host file, "
"from stdin"))
# if no host source is specified then read hosts from stdin
if not (args.hostfile or args.host):
args.hostfile = '-'
initial_manifest_tempfile = None
if args.manifest == '-':
# read initial manifest from stdin
import tempfile
try:
handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
handle, initial_manifest_temp_path = tempfile.mkstemp(
prefix='cdist.stdin.')
with os.fdopen(handle, 'w') as fd:
fd.write(sys.stdin.read())
except (IOError, OSError) as e:
raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
raise cdist.Error(("Creating tempfile for stdin data "
"failed: %s" % e))
args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path))
process = {}
failed_hosts = []
time_start = time.time()
@ -115,37 +118,38 @@ class Config(object):
hostcnt += 1
if args.parallel:
log.debug("Creating child process for %s", host)
process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True))
process[host] = multiprocessing.Process(
target=cls.onehost, args=(host, args, True))
process[host].start()
else:
try:
cls.onehost(host, args, parallel=False)
except cdist.Error as e:
failed_hosts.append(host)
# Catch errors in parallel mode when joining
if args.parallel:
for host in process.keys():
log.debug("Joining process %s", host)
process[host].join()
if not process[host].exitcode == 0:
failed_hosts.append(host)
time_end = time.time()
log.info("Total processing time for %s host(s): %s", hostcnt,
(time_end - time_start))
(time_end - time_start))
if len(failed_hosts) > 0:
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
raise cdist.Error("Failed to configure the following hosts: " +
" ".join(failed_hosts))
@classmethod
def onehost(cls, host, args, parallel):
"""Configure ONE system"""
log = logging.getLogger(host)
try:
local = cdist.exec.local.Local(
target_host=host,
@ -157,10 +161,10 @@ class Config(object):
target_host=host,
remote_exec=args.remote_exec,
remote_copy=args.remote_copy)
c = cls(local, remote, dry_run=args.dry_run)
c.run()
except cdist.Error as e:
log.error(e)
if parallel:
@ -168,7 +172,7 @@ class Config(object):
sys.exit(1)
else:
raise
except KeyboardInterrupt:
# Ignore in parallel mode, we are existing anyway
if parallel:
@ -188,49 +192,50 @@ class Config(object):
self.iterate_until_finished()
self.local.save_cache()
self.log.info("Finished successful run in %s seconds", time.time() - start_time)
self.log.info("Finished successful run in %s seconds",
time.time() - start_time)
def object_list(self):
"""Short name for object list retrieval"""
for cdist_object in core.CdistObject.list_objects(self.local.object_path,
self.local.type_path,
self.local.object_marker_name):
for cdist_object in core.CdistObject.list_objects(
self.local.object_path, self.local.type_path,
self.local.object_marker_name):
if cdist_object.cdist_type.is_install:
self.log.debug("Running in config mode, ignoring install object: {0}".format(cdist_object))
self.log.debug(("Running in config mode, ignoring install "
"object: {0}").format(cdist_object))
else:
yield cdist_object
def iterate_once(self):
"""
Iterate over the objects once - helper method for
Iterate over the objects once - helper method for
iterate_until_finished
"""
objects_changed = False
objects_changed = False
for cdist_object in self.object_list():
if cdist_object.requirements_unfinished(cdist_object.requirements):
"""We cannot do anything for this poor object"""
continue
if cdist_object.state == core.CdistObject.STATE_UNDEF:
"""Prepare the virgin object"""
self.object_prepare(cdist_object)
objects_changed = True
if cdist_object.requirements_unfinished(cdist_object.autorequire):
"""The previous step created objects we depend on - wait for them"""
"""The previous step created objects we depend on -
wait for them
"""
continue
if cdist_object.state == core.CdistObject.STATE_PREPARED:
self.object_run(cdist_object)
objects_changed = True
return objects_changed
def iterate_until_finished(self):
"""
Go through all objects and solve them
@ -256,22 +261,32 @@ class Config(object):
requirement_names = []
autorequire_names = []
for requirement in cdist_object.requirements_unfinished(cdist_object.requirements):
for requirement in cdist_object.requirements_unfinished(
cdist_object.requirements):
requirement_names.append(requirement.name)
for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire):
for requirement in cdist_object.requirements_unfinished(
cdist_object.autorequire):
autorequire_names.append(requirement.name)
requirements = "\n ".join(requirement_names)
autorequire = "\n ".join(autorequire_names)
info_string.append("%s requires:\n %s\n%s autorequires:\n %s" % (cdist_object.name, requirements, cdist_object.name, autorequire))
raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved:\n%s" %
("\n".join(info_string)))
autorequire = "\n ".join(autorequire_names)
info_string.append(("%s requires:\n"
" %s\n"
"%s ""autorequires:\n"
" %s" % (
cdist_object.name,
requirements, cdist_object.name,
autorequire)))
raise cdist.UnresolvableRequirementsError(
("The requirements of the following objects could not be "
"resolved:\n%s") % ("\n".join(info_string)))
def object_prepare(self, cdist_object):
"""Prepare object: Run type explorer + manifest"""
self.log.info("Running manifest and explorers for " + cdist_object.name)
self.log.info(
"Running manifest and explorers for " + cdist_object.name)
self.explorer.run_type_explorers(cdist_object)
self.manifest.run_type_manifest(cdist_object)
cdist_object.state = core.CdistObject.STATE_PREPARED
@ -281,7 +296,8 @@ class Config(object):
self.log.debug("Trying to run object %s" % (cdist_object.name))
if cdist_object.state == core.CdistObject.STATE_DONE:
raise cdist.Error("Attempting to run an already finished object: %s", cdist_object)
raise cdist.Error(("Attempting to run an already finished "
"object: %s"), cdist_object)
cdist_type = cdist_object.cdist_type
@ -304,7 +320,6 @@ class Config(object):
else:
self.log.info("Skipping code execution due to DRY RUN")
# Mark this object as done
self.log.debug("Finishing run of " + cdist_object.name)
cdist_object.state = core.CdistObject.STATE_DONE

14
cdist/core/__init__.py vendored

@ -20,10 +20,10 @@
#
#
from cdist.core.cdist_type import CdistType
from cdist.core.cdist_type import NoSuchTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code
from cdist.core.cdist_type import CdistType
from cdist.core.cdist_type import NoSuchTypeError
from cdist.core.cdist_object import CdistObject
from cdist.core.cdist_object import IllegalObjectIdError
from cdist.core.explorer import Explorer
from cdist.core.manifest import Manifest
from cdist.core.code import Code

82
cdist/core/cdist_object.py vendored

@ -32,6 +32,7 @@ from cdist.util import fsproperty
log = logging.getLogger(__name__)
class IllegalObjectIdError(cdist.Error):
def __init__(self, object_id, message=None):
self.object_id = object_id
@ -40,14 +41,17 @@ class IllegalObjectIdError(cdist.Error):
def __str__(self):
return '%s: %s' % (self.message, self.object_id)
class MissingObjectIdError(cdist.Error):
def __init__(self, type_name):
self.type_name = type_name
self.message = "Type %s requires object id (is not a singleton type)" % self.type_name
self.message = ("Type %s requires object id (is not a "
"singleton type)") % self.type_name
def __str__(self):
return '%s' % (self.message)
class CdistObject(object):
"""Represents a cdist object.
@ -64,7 +68,7 @@ class CdistObject(object):
STATE_DONE = "done"
def __init__(self, cdist_type, base_path, object_marker, object_id):
self.cdist_type = cdist_type # instance of Type
self.cdist_type = cdist_type # instance of Type
self.base_path = base_path
self.object_id = object_id
@ -74,7 +78,8 @@ class CdistObject(object):
self.sanitise_object_id()
self.name = self.join_name(self.cdist_type.name, self.object_id)
self.path = os.path.join(self.cdist_type.path, self.object_id, self.object_marker)
self.path = os.path.join(self.cdist_type.path, self.object_id,
self.object_marker)
self.absolute_path = os.path.join(self.base_path, self.path)
self.code_local_path = os.path.join(self.path, "code-local")
@ -84,12 +89,13 @@ class CdistObject(object):
@classmethod
def list_objects(cls, object_base_path, type_base_path, object_marker):
"""Return a list of object instances"""
for object_name in cls.list_object_names(object_base_path, object_marker):
for object_name in cls.list_object_names(
object_base_path, object_marker):
type_name, object_id = cls.split_name(object_name)
yield cls(cdist.core.CdistType(type_base_path, type_name),
base_path=object_base_path,
object_marker=object_marker,
object_id=object_id)
base_path=object_base_path,
object_marker=object_marker,
object_id=object_id)
@classmethod
def list_object_names(cls, object_base_path, object_marker):
@ -125,26 +131,34 @@ class CdistObject(object):
def validate_object_id(self):
if self.cdist_type.is_singleton and self.object_id:
raise IllegalObjectIdError('singleton objects can\'t have a object_id')
raise IllegalObjectIdError(('singleton objects can\'t have an '
'object_id'))
"""Validate the given object_id and raise IllegalObjectIdError if it's not valid.
"""Validate the given object_id and raise IllegalObjectIdError
if it's not valid.
"""
if self.object_id:
if self.object_marker in self.object_id.split(os.sep):
raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % self.object_marker)
raise IllegalObjectIdError(
self.object_id, ('object_id may not contain '
'\'%s\'') % self.object_marker)
if '//' in self.object_id:
raise IllegalObjectIdError(self.object_id, 'object_id may not contain //')
raise IllegalObjectIdError(
self.object_id, 'object_id may not contain //')
if self.object_id == '.':
raise IllegalObjectIdError(self.object_id, 'object_id may not be a .')
raise IllegalObjectIdError(
self.object_id, 'object_id may not be a .')
# If no object_id and type is not singleton => error out
if not self.object_id and not self.cdist_type.is_singleton:
raise MissingObjectIdError(self.cdist_type.name)
# Does not work: AttributeError: 'CdistObject' object has no attribute 'parameter_path'
# Does not work:
# AttributeError:
# 'CdistObject' object has no attribute 'parameter_path'
#"Type %s is not a singleton type - missing object id (parameters: %s)" %
# (self.cdist_type.name, self.parameters))
# "Type %s is not a singleton type - missing object id
# (parameters: %s)" % (self.cdist_type.name, self.parameters))
def object_from_name(self, object_name):
"""Convenience method for creating an object instance from an object name.
@ -152,7 +166,8 @@ class CdistObject(object):
Mainly intended to create objects when resolving requirements.
e.g:
<CdistObject __foo/bar>.object_from_name('__other/object') -> <CdistObject __other/object>
<CdistObject __foo/bar>.object_from_name('__other/object') ->
<CdistObject __other/object>
"""
@ -164,7 +179,8 @@ class CdistObject(object):
cdist_type = self.cdist_type.__class__(type_path, type_name)
return self.__class__(cdist_type, base_path, object_marker, object_id=object_id)
return self.__class__(cdist_type, base_path, object_marker,
object_id=object_id)
def __repr__(self):
return '<CdistObject %s>' % self.name
@ -172,7 +188,7 @@ class CdistObject(object):
def __eq__(self, other):
"""define equality as 'name is the same'"""
return self.name == other.name
def __hash__(self):
return hash(self.name)
@ -205,14 +221,22 @@ class CdistObject(object):
# return relative path
return os.path.join(self.path, "explorer")
requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require'))
autorequire = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path))
explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path))
state = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.absolute_path, "state"))
source = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, "source"))
code_local = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_local_path))
code_remote = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
requirements = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, 'require'))
autorequire = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, 'autorequire'))
parameters = fsproperty.DirectoryDictProperty(
lambda obj: os.path.join(obj.base_path, obj.parameter_path))
explorers = fsproperty.DirectoryDictProperty(
lambda obj: os.path.join(obj.base_path, obj.explorer_path))
state = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.absolute_path, "state"))
source = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, "source"))
code_local = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.base_path, obj.code_local_path))
code_remote = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
@property
def exists(self):
@ -224,10 +248,12 @@ class CdistObject(object):
"""
try:
os.makedirs(self.absolute_path, exist_ok=allow_overwrite)
absolute_parameter_path = os.path.join(self.base_path, self.parameter_path)
absolute_parameter_path = os.path.join(self.base_path,
self.parameter_path)
os.makedirs(absolute_parameter_path, exist_ok=allow_overwrite)
except EnvironmentError as error:
raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error))
raise cdist.Error(('Error creating directories for cdist object: '
'%s: %s') % (self, error))
def requirements_unfinished(self, requirements):
"""Return state whether requirements are satisfied"""

38
cdist/core/cdist_type.py vendored

@ -24,6 +24,7 @@ import os
import cdist
class NoSuchTypeError(cdist.Error):
def __init__(self, name, type_path, type_absolute_path):
self.name = name
@ -31,7 +32,8 @@ class NoSuchTypeError(cdist.Error):
self.type_absolute_path = type_absolute_path
def __str__(self):
return "Type '%s' does not exist at %s" % (self.type_path, self.type_absolute_path)
return "Type '%s' does not exist at %s" % (
self.type_path, self.type_absolute_path)
class CdistType(object):
@ -75,13 +77,13 @@ class CdistType(object):
"""Return a list of type names"""
return os.listdir(base_path)
_instances = {}
def __new__(cls, *args, **kwargs):
"""only one instance of each named type may exist"""
# name is second argument
name = args[1]
if not name in cls._instances:
if name not in cls._instances:
instance = super(CdistType, cls).__new__(cls)
cls._instances[name] = instance
# return instance so __init__ is called
@ -103,7 +105,8 @@ class CdistType(object):
@property
def is_install(self):
"""Check whether a type is used for installation (if not: for configuration)"""
"""Check whether a type is used for installation
(if not: for configuration)"""
return os.path.isfile(os.path.join(self.absolute_path, "install"))
@property
@ -111,7 +114,8 @@ class CdistType(object):
"""Return a list of available explorers"""
if not self.__explorers:
try:
self.__explorers = os.listdir(os.path.join(self.absolute_path, "explorer"))
self.__explorers = os.listdir(os.path.join(self.absolute_path,
"explorer"))
except EnvironmentError:
# error ignored
self.__explorers = []
@ -123,7 +127,9 @@ class CdistType(object):
if not self.__required_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "required")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"required")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -139,7 +145,9 @@ class CdistType(object):
if not self.__required_multiple_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "required_multiple")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"required_multiple")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -155,7 +163,9 @@ class CdistType(object):
if not self.__optional_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "optional")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"optional")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -171,7 +181,9 @@ class CdistType(object):
if not self.__optional_multiple_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "optional_multiple")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"optional_multiple")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -187,7 +199,9 @@ class CdistType(object):
if not self.__boolean_parameters:
parameters = []
try:
with open(os.path.join(self.absolute_path, "parameter", "boolean")) as fd:
with open(os.path.join(self.absolute_path,
"parameter",
"boolean")) as fd:
for line in fd:
parameters.append(line.strip())
except EnvironmentError:
@ -202,7 +216,9 @@ class CdistType(object):
if not self.__parameter_defaults:
defaults = {}
try:
defaults_dir = os.path.join(self.absolute_path, "parameter", "default")
defaults_dir = os.path.join(self.absolute_path,
"parameter",
"default")
for name in os.listdir(defaults_dir):
try:
with open(os.path.join(defaults_dir, name)) as fd:

43
cdist/core/code.py vendored

@ -37,15 +37,17 @@ common:
PATH: prepend directory with type emulator symlinks == local.bin_path
__target_host: the target host we are working on
__cdist_manifest: full qualified path of the manifest == script
__cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator
== local.type_path
__cdist_type_base_path: full qualified path to the directory where
types are defined for use in type emulator
== local.type_path
gencode-local
script: full qualified path to a types gencode-local
env:
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__global: full qualified path to the global
output dir == local.out_path
__object: full qualified path to the object's dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
@ -59,7 +61,8 @@ gencode-remote
env:
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__global: full qualified path to the global
output dir == local.out_path
__object: full qualified path to the object's dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
@ -98,7 +101,8 @@ class Code(object):
def _run_gencode(self, cdist_object, which):
cdist_type = cdist_object.cdist_type
script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which))
script = os.path.join(self.local.type_path,
getattr(cdist_type, 'gencode_%s_path' % which))
if os.path.isfile(script):
env = os.environ.copy()
env.update(self.env)
@ -108,8 +112,9 @@ class Code(object):
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
})
message_prefix=cdist_object.name
return self.local.run_script(script, env=env, return_output=True, message_prefix=message_prefix)
message_prefix = cdist_object.name
return self.local.run_script(script, env=env, return_output=True,
message_prefix=message_prefix)
def run_gencode_local(self, cdist_object):
"""Run the gencode-local script for the given cdist object."""
@ -120,21 +125,26 @@ class Code(object):
return self._run_gencode(cdist_object, 'remote')
def transfer_code_remote(self, cdist_object):
"""Transfer the code_remote script for the given object to the remote side."""
source = os.path.join(self.local.object_path, cdist_object.code_remote_path)
destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path)
"""Transfer the code_remote script for the given object to the
remote side."""
source = os.path.join(self.local.object_path,
cdist_object.code_remote_path)
destination = os.path.join(self.remote.object_path,
cdist_object.code_remote_path)
# FIXME: BUG: do not create destination, but top level of destination!
self.remote.mkdir(destination)
self.remote.transfer(source, destination)
def _run_code(self, cdist_object, which, env=None):
which_exec = getattr(self, which)
script = os.path.join(which_exec.object_path, getattr(cdist_object, 'code_%s_path' % which))
script = os.path.join(which_exec.object_path,
getattr(cdist_object, 'code_%s_path' % which))
return which_exec.run_script(script, env=env)
def run_code_local(self, cdist_object):
"""Run the code-local script for the given cdist object."""
# Put some env vars, to allow read only access to the parameters over $__object
# Put some env vars, to allow read only access to the parameters
# over $__object
env = os.environ.copy()
env.update(self.env)
env.update({
@ -144,10 +154,13 @@ class Code(object):
return self._run_code(cdist_object, 'local', env=env)
def run_code_remote(self, cdist_object):
"""Run the code-remote script for the given cdist object on the remote side."""
# Put some env vars, to allow read only access to the parameters over $__object which is already on the remote side
"""Run the code-remote script for the given cdist object on the
remote side."""
# Put some env vars, to allow read only access to the parameters
# over $__object which is already on the remote side
env = {
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object': os.path.join(self.remote.object_path,
cdist_object.path),
'__object_id': cdist_object.object_id,
}
return self._run_code(cdist_object, 'remote', env=env)

55
cdist/core/explorer.py vendored

@ -31,7 +31,8 @@ common:
runs only remotely, needs local and remote to construct paths
env:
__explorer: full qualified path to other global explorers on remote side
__explorer: full qualified path to other global explorers on
remote side
-> remote.global_explorer_path
a global explorer is:
@ -52,7 +53,8 @@ type explorer is:
__object: full qualified path to the object's remote dir
__object_id: the objects id
__object_fq: full qualified object id, iow: $type.name + / + object_id
__type_explorer: full qualified path to the other type explorers on remote side
__type_explorer: full qualified path to the other type explorers on
remote side
creates: nothing, returns output
@ -76,7 +78,7 @@ class Explorer(object):
}
self._type_explorers_transferred = []
### global
# global
def list_global_explorer_names(self):
"""Return a list of global explorer names."""
@ -98,15 +100,17 @@ class Explorer(object):
def transfer_global_explorers(self):
"""Transfer the global explorers to the remote side."""
self.remote.mkdir(self.remote.global_explorer_path)
self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path)
self.remote.run(["chmod", "0700", "%s/*" % (self.remote.global_explorer_path)])
self.remote.transfer(self.local.global_explorer_path,
self.remote.global_explorer_path)
self.remote.run(["chmod", "0700",
"%s/*" % (self.remote.global_explorer_path)])
def run_global_explorer(self, explorer):
"""Run the given global explorer and return it's output."""
script = os.path.join(self.remote.global_explorer_path, explorer)
return self.remote.run_script(script, env=self.env, return_output=True)
### type
# type
def list_type_explorer_names(self, cdist_type):
"""Return a list of explorer names for the given type."""
@ -121,37 +125,48 @@ class Explorer(object):
in the object.
"""
self.log.debug("Transfering type explorers for type: %s", cdist_object.cdist_type)
self.log.debug("Transfering type explorers for type: %s",
cdist_object.cdist_type)
self.transfer_type_explorers(cdist_object.cdist_type)
self.log.debug("Transfering object parameters for object: %s", cdist_object.name)
self.log.debug("Transfering object parameters for object: %s",
cdist_object.name)
self.transfer_object_parameters(cdist_object)
for explorer in self.list_type_explorer_names(cdist_object.cdist_type):
output = self.run_type_explorer(explorer, cdist_object)
self.log.debug("Running type explorer '%s' for object '%s'", explorer, cdist_object.name)
self.log.debug("Running type explorer '%s' for object '%s'",
explorer, cdist_object.name)
cdist_object.explorers[explorer] = output
def run_type_explorer(self, explorer, cdist_object):
"""Run the given type explorer for the given object and return it's output."""
"""Run the given type explorer for the given object and return
it's output."""
cdist_type = cdist_object.cdist_type
env = self.env.copy()
env.update({
'__object': os.path.join(self.remote.object_path, cdist_object.path),
'__object': os.path.join(self.remote.object_path,
cdist_object.path),
'__object_id': cdist_object.object_id,
'__object_name': cdist_object.name,
'__object_fq': cdist_object.path,
'__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path)
'__type_explorer': os.path.join(self.remote.type_path,
cdist_type.explorer_path)
})
script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer)
script = os.path.join(self.remote.type_path, cdist_type.explorer_path,
explorer)
return self.remote.run_script(script, env=env, return_output=True)
def transfer_type_explorers(self, cdist_type):
"""Transfer the type explorers for the given type to the remote side."""
"""Transfer the type explorers for the given type to the
remote side."""
if cdist_type.explorers:
if cdist_type.name in self._type_explorers_transferred:
self.log.debug("Skipping retransfer of type explorers for: %s", cdist_type)
self.log.debug("Skipping retransfer of type explorers for: %s",
cdist_type)
else:
source = os.path.join(self.local.type_path, cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path, cdist_type.explorer_path)
source = os.path.join(self.local.type_path,
cdist_type.explorer_path)
destination = os.path.join(self.remote.type_path,
cdist_type.explorer_path)
self.remote.mkdir(destination)
self.remote.transfer(source, destination)
self.remote.run(["chmod", "0700", "%s/*" % (destination)])
@ -160,7 +175,9 @@ class Explorer(object):
def transfer_object_parameters(self, cdist_object):
"""Transfer the parameters for the given object to the remote side."""
if cdist_object.parameters:
source = os.path.join(self.local.object_path, cdist_object.parameter_path)
destination = os.path.join(self.remote.object_path, cdist_object.parameter_path)
source = os.path.join(self.local.object_path,
cdist_object.parameter_path)
destination = os.path.join(self.remote.object_path,
cdist_object.parameter_path)
self.remote.mkdir(destination)
self.remote.transfer(source, destination)

34
cdist/core/manifest.py vendored

@ -32,9 +32,11 @@ common:
env:
PATH: prepend directory with type emulator symlinks == local.bin_path
__target_host: the target host we are working on
__global: full qualified path to the global output dir == local.out_path
__global: full qualified path to the global
output dir == local.out_path
__cdist_manifest: full qualified path of the manifest == script
__cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator
__cdist_type_base_path: full qualified path to the directory where
types are defined for use in type emulator
== local.type_path
__files: full qualified path to the files dir
@ -58,6 +60,7 @@ type manifeste is:
creates: new objects through type emulator
'''
class NoInitialManifestError(cdist.Error):
"""
Display missing initial manifest:
@ -72,7 +75,9 @@ class NoInitialManifestError(cdist.Error):
if user_supplied:
if os.path.islink(manifest_path):
self.message = "%s: %s -> %s" % (msg_header, manifest_path, os.path.realpath(manifest_path))
self.message = "%s: %s -> %s" % (
msg_header, manifest_path,
os.path.realpath(manifest_path))
else:
self.message = "%s: %s" % (msg_header, manifest_path)
else:
@ -94,14 +99,15 @@ class Manifest(object):
self.env = {
'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']),
'__cdist_type_base_path': self.local.type_path, # for use in type emulator
# for use in type emulator
'__cdist_type_base_path': self.local.type_path,
'__global': self.local.base_path,
'__target_host': self.target_host,
'__files': self.local.files_path,
}
if self.log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__cdist_debug': "yes" })
if self.log.getEffectiveLevel() == logging.DEBUG:
self.env.update({'__cdist_debug': "yes"})
def env_initial_manifest(self, initial_manifest):
env = os.environ.copy()
@ -124,11 +130,14 @@ class Manifest(object):
if not os.path.isfile(initial_manifest):
raise NoInitialManifestError(initial_manifest, user_supplied)
message_prefix="initialmanifest"
self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix)
message_prefix = "initialmanifest"
self.local.run_script(initial_manifest,
env=self.env_initial_manifest(initial_manifest),
message_prefix=message_prefix)
def env_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
type_manifest = os.path.join(self.local.type_path,
cdist_object.cdist_type.manifest_path)
env = os.environ.copy()
env.update(self.env)
env.update({
@ -143,7 +152,10 @@ class Manifest(object):
return env
def run_type_manifest(self, cdist_object):
type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path)
type_manifest = os.path.join(self.local.type_path,
cdist_object.cdist_type.manifest_path)
message_prefix = cdist_object.name
if os.path.isfile(type_manifest):
self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix)
self.local.run_script(type_manifest,
env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix)

119
cdist/emulator.py vendored

@ -29,10 +29,12 @@ import sys
import cdist
from cdist import core
class MissingRequiredEnvironmentVariableError(cdist.Error):
def __init__(self, name):
self.name = name
self.message = "Emulator requires the environment variable %s to be setup" % self.name
self.message = ("Emulator requires the environment variable %s to be "
"setup" % self.name)
def __str__(self):
return self.message
@ -41,7 +43,7 @@ class MissingRequiredEnvironmentVariableError(cdist.Error):
class DefaultList(list):
"""Helper class to allow default values for optional_multiple parameters.
@see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ
@see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ
"""
def __copy__(self):
return []
@ -54,20 +56,20 @@ class DefaultList(list):
class Emulator(object):
def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
self.argv = argv
self.stdin = stdin
self.env = env
self.argv = argv
self.stdin = stdin
self.env = env
self.object_id = ''
self.object_id = ''
try:
self.global_path = self.env['__global']
self.target_host = self.env['__target_host']
self.global_path = self.env['__global']
self.target_host = self.env['__target_host']
# Internal variables
self.object_source = self.env['__cdist_manifest']
self.object_source = self.env['__cdist_manifest']
self.type_base_path = self.env['__cdist_type_base_path']
self.object_marker = self.env['__cdist_object_marker']
self.object_marker = self.env['__cdist_object_marker']
except KeyError as e:
raise MissingRequiredEnvironmentVariableError(e.args[0])
@ -75,8 +77,8 @@ class Emulator(object):
self.object_base_path = os.path.join(self.global_path, "object")
self.typeorder_path = os.path.join(self.global_path, "typeorder")
self.type_name = os.path.basename(argv[0])
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
self.type_name = os.path.basename(argv[0])
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
self.__init_log()
@ -88,7 +90,8 @@ class Emulator(object):
self.save_stdin()
self.record_requirements()
self.record_auto_requirements()
self.log.debug("Finished %s %s" % (self.cdist_object.path, self.parameters))
self.log.debug("Finished %s %s" % (
self.cdist_object.path, self.parameters))
def __init_log(self):
"""Setup logging facility"""
@ -98,30 +101,38 @@ class Emulator(object):
else:
logging.root.setLevel(logging.INFO)
self.log = logging.getLogger(self.target_host)
self.log = logging.getLogger(self.target_host)
def commandline(self):
"""Parse command line"""
parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS)
parser = argparse.ArgumentParser(add_help=False,
argument_default=argparse.SUPPRESS)
for parameter in self.cdist_type.required_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='store', required=True)
parser.add_argument(argument, dest=parameter, action='store',
required=True)
for parameter in self.cdist_type.required_multiple_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='append', required=True)
parser.add_argument(argument, dest=parameter, action='append',
required=True)
for parameter in self.cdist_type.optional_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='store', required=False,
default=self.cdist_type.parameter_defaults.get(parameter, None))
default = self.cdist_type.parameter_defaults.get(parameter, None)
parser.add_argument(argument, dest=parameter, action='store',
required=False, default=default)
for parameter in self.cdist_type.optional_multiple_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='append', required=False,
default=DefaultList.create(self.cdist_type.parameter_defaults.get(parameter, None)))
default = DefaultList.create(
self.cdist_type.parameter_defaults.get(
parameter, None))
parser.add_argument(argument, dest=parameter, action='append',
required=False, default=default)
for parameter in self.cdist_type.boolean_parameters:
argument = "--" + parameter
parser.add_argument(argument, dest=parameter, action='store_const', const='')
parser.add_argument(argument, dest=parameter,
action='store_const', const='')
# If not singleton support one positional parameter
if not self.cdist_type.is_singleton:
@ -140,30 +151,33 @@ class Emulator(object):
del self.args.object_id
# Instantiate the cdist object we are defining
self.cdist_object = core.CdistObject(self.cdist_type,
self.object_base_path, self.object_marker, self.object_id)
self.cdist_object = core.CdistObject(
self.cdist_type, self.object_base_path, self.object_marker,
self.object_id)
# Create object with given parameters
self.parameters = {}
for key,value in vars(self.args).items():
for key, value in vars(self.args).items():
if value is not None:
self.parameters[key] = value
if self.cdist_object.exists and not 'CDIST_OVERRIDE' in self.env:
if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
# make existing requirements a set, so we can compare it
# later with new requirements
if self.cdist_object.parameters != self.parameters:
errmsg = ("Object %s already exists with conflicting "
"parameters:\n%s: %s\n%s: %s" % (self.cdist_object.name,
" ".join(self.cdist_object.source),
self.cdist_object.parameters,
self.object_source,
self.parameters))
"parameters:\n%s: %s\n%s: %s" % (
self.cdist_object.name,
" ".join(self.cdist_object.source),
self.cdist_object.parameters,
self.object_source,
self.parameters))
self.log.error(errmsg)
raise cdist.Error(errmsg)
else:
if self.cdist_object.exists:
self.log.debug('Object %s override forced with CDIST_OVERRIDE',self.cdist_object.name)
self.log.debug(('Object %s override forced with '
'CDIST_OVERRIDE'), self.cdist_object.name)
self.cdist_object.create(True)
else:
self.cdist_object.create()
@ -176,6 +190,7 @@ class Emulator(object):
self.cdist_object.source.append(self.object_source)
chunk_size = 65536
def _read_stdin(self):
return self.stdin.read(self.chunk_size)
@ -197,7 +212,6 @@ class Emulator(object):
except EnvironmentError as e:
raise cdist.Error('Failed to read from stdin: %s' % e)
def record_requirement(self, requirement):
"""record requirement and return recorded requirement"""
@ -206,13 +220,15 @@ class Emulator(object):
cdist_object = self.cdist_object.object_from_name(requirement)
except core.cdist_type.NoSuchTypeError as e:
self.log.error(("%s requires object %s, but type %s does not"
" exist. Defined at %s" % (self.cdist_object.name,
requirement, e.name, self.object_source)))
" exist. Defined at %s" % (
self.cdist_object.name,
requirement, e.name, self.object_source)))
raise
except core.cdist_object.MissingObjectIdError as e:
self.log.error(("%s requires object %s without object id."
" Defined at %s" % (self.cdist_object.name, requirement,
self.object_source)))
" Defined at %s" % (self.cdist_object.name,
requirement,
self.object_source)))
raise
self.log.debug("Recording requirement: %s", requirement)
@ -224,12 +240,13 @@ class Emulator(object):
return cdist_object.name
def record_requirements(self):
"""record requirements"""
# Inject the predecessor, but not if its an override (this would leed to an circular dependency)
if "CDIST_ORDER_DEPENDENCY" in self.env and not 'CDIST_OVERRIDE' in self.env:
# Inject the predecessor, but not if its an override
# (this would leed to an circular dependency)
if ("CDIST_ORDER_DEPENDENCY" in self.env and
'CDIST_OVERRIDE' not in self.env):
# load object name created bevor this one from typeorder file ...
with open(self.typeorder_path, 'r') as typecreationfile:
typecreationorder = typecreationfile.readlines()
@ -240,9 +257,12 @@ class Emulator(object):
self.env['require'] += " " + lastcreatedtype
else:
self.env['require'] = lastcreatedtype
self.log.debug("Injecting require for CDIST_ORDER_DEPENDENCY: %s for %s", lastcreatedtype, self.cdist_object.name)
self.log.debug(("Injecting require for "
"CDIST_ORDER_DEPENDENCY: %s for %s"),
lastcreatedtype, self.cdist_object.name)
except IndexError:
# if no second last line, we are on the first type, so do not set a requirement
# if no second last line, we are on the first type,
# so do not set a requirement
pass
if "require" in self.env: