diff --git a/cdist/__init__.py b/cdist/__init__.py index e789499c..b7436b5a 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -44,25 +44,30 @@ BANNER = """ REMOTE_COPY = "scp -o User=root" REMOTE_EXEC = "ssh -o User=root" + class Error(Exception): """Base exception class for this project""" pass + class UnresolvableRequirementsError(cdist.Error): """Resolving requirements failed""" pass + class CdistObjectError(Error): """Something went wrong with an object""" - + def __init__(self, cdist_object, message): self.name = cdist_object.name self.source = " ".join(cdist_object.source) self.message = message - def __str__(self): - return '%s: %s (defined at %s)' % (self.name, self.message, self.source) + return '%s: %s (defined at %s)' % (self.name, + self.message, + self.source) + def file_to_list(filename): """Return list from \n seperated file""" diff --git a/cdist/config.py b/cdist/config.py index f5e62ce1..5ebeab14 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -35,19 +35,21 @@ import cdist.exec.remote from cdist import core + class Config(object): """Cdist main class to hold arbitrary data""" def __init__(self, local, remote, dry_run=False): - self.local = local - self.remote = remote - self.log = logging.getLogger(self.local.target_host) - self.dry_run = dry_run + self.local = local + self.remote = remote + self.log = logging.getLogger(self.local.target_host) + self.dry_run = dry_run - self.explorer = core.Explorer(self.local.target_host, self.local, self.remote) + self.explorer = core.Explorer(self.local.target_host, self.local, + self.remote) self.manifest = core.Manifest(self.local.target_host, self.local) - self.code = core.Code(self.local.target_host, self.local, self.remote) + self.code = core.Code(self.local.target_host, self.local, self.remote) def _init_files_dirs(self): """Prepare files and directories for the run""" @@ -65,7 +67,7 @@ class Config(object): try: for host in fileinput.input(files=(source)): # remove leading and trailing whitespace - yield host.strip() + yield host.strip() except (IOError, OSError) as e: raise cdist.Error("Error reading hosts from \'{}\'".format( source)) @@ -74,7 +76,6 @@ class Config(object): for host in source: yield host - @classmethod def commandline(cls, args): """Configure remote system""" @@ -84,27 +85,29 @@ class Config(object): log = logging.getLogger("cdist") if args.manifest == '-' and args.hostfile == '-': - raise cdist.Error(("Cannot read both, manifest and host file, " - "from stdin")) + raise cdist.Error(("Cannot read both, manifest and host file, " + "from stdin")) # if no host source is specified then read hosts from stdin if not (args.hostfile or args.host): args.hostfile = '-' - + initial_manifest_tempfile = None if args.manifest == '-': # read initial manifest from stdin import tempfile try: - handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.') + handle, initial_manifest_temp_path = tempfile.mkstemp( + prefix='cdist.stdin.') with os.fdopen(handle, 'w') as fd: fd.write(sys.stdin.read()) except (IOError, OSError) as e: - raise cdist.Error("Creating tempfile for stdin data failed: %s" % e) - + raise cdist.Error(("Creating tempfile for stdin data " + "failed: %s" % e)) + args.manifest = initial_manifest_temp_path import atexit atexit.register(lambda: os.remove(initial_manifest_temp_path)) - + process = {} failed_hosts = [] time_start = time.time() @@ -115,37 +118,38 @@ class Config(object): hostcnt += 1 if args.parallel: log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=cls.onehost, args=(host, args, True)) + process[host] = multiprocessing.Process( + target=cls.onehost, args=(host, args, True)) process[host].start() else: try: cls.onehost(host, args, parallel=False) except cdist.Error as e: failed_hosts.append(host) - + # Catch errors in parallel mode when joining if args.parallel: for host in process.keys(): log.debug("Joining process %s", host) process[host].join() - + if not process[host].exitcode == 0: failed_hosts.append(host) - + time_end = time.time() log.info("Total processing time for %s host(s): %s", hostcnt, - (time_end - time_start)) - + (time_end - time_start)) + if len(failed_hosts) > 0: - raise cdist.Error("Failed to configure the following hosts: " + - " ".join(failed_hosts)) - + raise cdist.Error("Failed to configure the following hosts: " + + " ".join(failed_hosts)) + @classmethod def onehost(cls, host, args, parallel): """Configure ONE system""" log = logging.getLogger(host) - + try: local = cdist.exec.local.Local( target_host=host, @@ -157,10 +161,10 @@ class Config(object): target_host=host, remote_exec=args.remote_exec, remote_copy=args.remote_copy) - + c = cls(local, remote, dry_run=args.dry_run) c.run() - + except cdist.Error as e: log.error(e) if parallel: @@ -168,7 +172,7 @@ class Config(object): sys.exit(1) else: raise - + except KeyboardInterrupt: # Ignore in parallel mode, we are existing anyway if parallel: @@ -188,49 +192,50 @@ class Config(object): self.iterate_until_finished() self.local.save_cache() - self.log.info("Finished successful run in %s seconds", time.time() - start_time) - + self.log.info("Finished successful run in %s seconds", + time.time() - start_time) def object_list(self): """Short name for object list retrieval""" - for cdist_object in core.CdistObject.list_objects(self.local.object_path, - self.local.type_path, - self.local.object_marker_name): + for cdist_object in core.CdistObject.list_objects( + self.local.object_path, self.local.type_path, + self.local.object_marker_name): if cdist_object.cdist_type.is_install: - self.log.debug("Running in config mode, ignoring install object: {0}".format(cdist_object)) + self.log.debug(("Running in config mode, ignoring install " + "object: {0}").format(cdist_object)) else: yield cdist_object - def iterate_once(self): """ - Iterate over the objects once - helper method for + Iterate over the objects once - helper method for iterate_until_finished """ - objects_changed = False + objects_changed = False for cdist_object in self.object_list(): if cdist_object.requirements_unfinished(cdist_object.requirements): """We cannot do anything for this poor object""" continue - + if cdist_object.state == core.CdistObject.STATE_UNDEF: """Prepare the virgin object""" - + self.object_prepare(cdist_object) objects_changed = True - + if cdist_object.requirements_unfinished(cdist_object.autorequire): - """The previous step created objects we depend on - wait for them""" + """The previous step created objects we depend on - + wait for them + """ continue - + if cdist_object.state == core.CdistObject.STATE_PREPARED: self.object_run(cdist_object) objects_changed = True return objects_changed - def iterate_until_finished(self): """ Go through all objects and solve them @@ -256,22 +261,32 @@ class Config(object): requirement_names = [] autorequire_names = [] - for requirement in cdist_object.requirements_unfinished(cdist_object.requirements): + for requirement in cdist_object.requirements_unfinished( + cdist_object.requirements): requirement_names.append(requirement.name) - for requirement in cdist_object.requirements_unfinished(cdist_object.autorequire): + for requirement in cdist_object.requirements_unfinished( + cdist_object.autorequire): autorequire_names.append(requirement.name) requirements = "\n ".join(requirement_names) - autorequire = "\n ".join(autorequire_names) - info_string.append("%s requires:\n %s\n%s autorequires:\n %s" % (cdist_object.name, requirements, cdist_object.name, autorequire)) + autorequire = "\n ".join(autorequire_names) + info_string.append(("%s requires:\n" + " %s\n" + "%s ""autorequires:\n" + " %s" % ( + cdist_object.name, + requirements, cdist_object.name, + autorequire))) - raise cdist.UnresolvableRequirementsError("The requirements of the following objects could not be resolved:\n%s" % - ("\n".join(info_string))) + raise cdist.UnresolvableRequirementsError( + ("The requirements of the following objects could not be " + "resolved:\n%s") % ("\n".join(info_string))) def object_prepare(self, cdist_object): """Prepare object: Run type explorer + manifest""" - self.log.info("Running manifest and explorers for " + cdist_object.name) + self.log.info( + "Running manifest and explorers for " + cdist_object.name) self.explorer.run_type_explorers(cdist_object) self.manifest.run_type_manifest(cdist_object) cdist_object.state = core.CdistObject.STATE_PREPARED @@ -281,7 +296,8 @@ class Config(object): self.log.debug("Trying to run object %s" % (cdist_object.name)) if cdist_object.state == core.CdistObject.STATE_DONE: - raise cdist.Error("Attempting to run an already finished object: %s", cdist_object) + raise cdist.Error(("Attempting to run an already finished " + "object: %s"), cdist_object) cdist_type = cdist_object.cdist_type @@ -304,7 +320,6 @@ class Config(object): else: self.log.info("Skipping code execution due to DRY RUN") - # Mark this object as done self.log.debug("Finishing run of " + cdist_object.name) cdist_object.state = core.CdistObject.STATE_DONE diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index d773fc01..41e00a3a 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -20,10 +20,10 @@ # # -from cdist.core.cdist_type import CdistType -from cdist.core.cdist_type import NoSuchTypeError -from cdist.core.cdist_object import CdistObject -from cdist.core.cdist_object import IllegalObjectIdError -from cdist.core.explorer import Explorer -from cdist.core.manifest import Manifest -from cdist.core.code import Code +from cdist.core.cdist_type import CdistType +from cdist.core.cdist_type import NoSuchTypeError +from cdist.core.cdist_object import CdistObject +from cdist.core.cdist_object import IllegalObjectIdError +from cdist.core.explorer import Explorer +from cdist.core.manifest import Manifest +from cdist.core.code import Code diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 8c6ee9c9..262db8bf 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -32,6 +32,7 @@ from cdist.util import fsproperty log = logging.getLogger(__name__) + class IllegalObjectIdError(cdist.Error): def __init__(self, object_id, message=None): self.object_id = object_id @@ -40,14 +41,17 @@ class IllegalObjectIdError(cdist.Error): def __str__(self): return '%s: %s' % (self.message, self.object_id) + class MissingObjectIdError(cdist.Error): def __init__(self, type_name): self.type_name = type_name - self.message = "Type %s requires object id (is not a singleton type)" % self.type_name + self.message = ("Type %s requires object id (is not a " + "singleton type)") % self.type_name def __str__(self): return '%s' % (self.message) + class CdistObject(object): """Represents a cdist object. @@ -64,7 +68,7 @@ class CdistObject(object): STATE_DONE = "done" def __init__(self, cdist_type, base_path, object_marker, object_id): - self.cdist_type = cdist_type # instance of Type + self.cdist_type = cdist_type # instance of Type self.base_path = base_path self.object_id = object_id @@ -74,7 +78,8 @@ class CdistObject(object): self.sanitise_object_id() self.name = self.join_name(self.cdist_type.name, self.object_id) - self.path = os.path.join(self.cdist_type.path, self.object_id, self.object_marker) + self.path = os.path.join(self.cdist_type.path, self.object_id, + self.object_marker) self.absolute_path = os.path.join(self.base_path, self.path) self.code_local_path = os.path.join(self.path, "code-local") @@ -84,12 +89,13 @@ class CdistObject(object): @classmethod def list_objects(cls, object_base_path, type_base_path, object_marker): """Return a list of object instances""" - for object_name in cls.list_object_names(object_base_path, object_marker): + for object_name in cls.list_object_names( + object_base_path, object_marker): type_name, object_id = cls.split_name(object_name) yield cls(cdist.core.CdistType(type_base_path, type_name), - base_path=object_base_path, - object_marker=object_marker, - object_id=object_id) + base_path=object_base_path, + object_marker=object_marker, + object_id=object_id) @classmethod def list_object_names(cls, object_base_path, object_marker): @@ -125,26 +131,34 @@ class CdistObject(object): def validate_object_id(self): if self.cdist_type.is_singleton and self.object_id: - raise IllegalObjectIdError('singleton objects can\'t have a object_id') + raise IllegalObjectIdError(('singleton objects can\'t have an ' + 'object_id')) - """Validate the given object_id and raise IllegalObjectIdError if it's not valid. + """Validate the given object_id and raise IllegalObjectIdError + if it's not valid. """ if self.object_id: if self.object_marker in self.object_id.split(os.sep): - raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % self.object_marker) + raise IllegalObjectIdError( + self.object_id, ('object_id may not contain ' + '\'%s\'') % self.object_marker) if '//' in self.object_id: - raise IllegalObjectIdError(self.object_id, 'object_id may not contain //') + raise IllegalObjectIdError( + self.object_id, 'object_id may not contain //') if self.object_id == '.': - raise IllegalObjectIdError(self.object_id, 'object_id may not be a .') + raise IllegalObjectIdError( + self.object_id, 'object_id may not be a .') # If no object_id and type is not singleton => error out if not self.object_id and not self.cdist_type.is_singleton: raise MissingObjectIdError(self.cdist_type.name) - # Does not work: AttributeError: 'CdistObject' object has no attribute 'parameter_path' + # Does not work: + # AttributeError: + # 'CdistObject' object has no attribute 'parameter_path' - #"Type %s is not a singleton type - missing object id (parameters: %s)" % - # (self.cdist_type.name, self.parameters)) + # "Type %s is not a singleton type - missing object id + # (parameters: %s)" % (self.cdist_type.name, self.parameters)) def object_from_name(self, object_name): """Convenience method for creating an object instance from an object name. @@ -152,7 +166,8 @@ class CdistObject(object): Mainly intended to create objects when resolving requirements. e.g: - .object_from_name('__other/object') -> + .object_from_name('__other/object') -> + """ @@ -164,7 +179,8 @@ class CdistObject(object): cdist_type = self.cdist_type.__class__(type_path, type_name) - return self.__class__(cdist_type, base_path, object_marker, object_id=object_id) + return self.__class__(cdist_type, base_path, object_marker, + object_id=object_id) def __repr__(self): return '' % self.name @@ -172,7 +188,7 @@ class CdistObject(object): def __eq__(self, other): """define equality as 'name is the same'""" return self.name == other.name - + def __hash__(self): return hash(self.name) @@ -205,14 +221,22 @@ class CdistObject(object): # return relative path return os.path.join(self.path, "explorer") - requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require')) - autorequire = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'autorequire')) - parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.parameter_path)) - explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path)) - state = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.absolute_path, "state")) - source = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, "source")) - code_local = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_local_path)) - code_remote = fsproperty.FileStringProperty(lambda obj: os.path.join(obj.base_path, obj.code_remote_path)) + requirements = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'require')) + autorequire = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'autorequire')) + parameters = fsproperty.DirectoryDictProperty( + lambda obj: os.path.join(obj.base_path, obj.parameter_path)) + explorers = fsproperty.DirectoryDictProperty( + lambda obj: os.path.join(obj.base_path, obj.explorer_path)) + state = fsproperty.FileStringProperty( + lambda obj: os.path.join(obj.absolute_path, "state")) + source = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, "source")) + code_local = fsproperty.FileStringProperty( + lambda obj: os.path.join(obj.base_path, obj.code_local_path)) + code_remote = fsproperty.FileStringProperty( + lambda obj: os.path.join(obj.base_path, obj.code_remote_path)) @property def exists(self): @@ -224,10 +248,12 @@ class CdistObject(object): """ try: os.makedirs(self.absolute_path, exist_ok=allow_overwrite) - absolute_parameter_path = os.path.join(self.base_path, self.parameter_path) + absolute_parameter_path = os.path.join(self.base_path, + self.parameter_path) os.makedirs(absolute_parameter_path, exist_ok=allow_overwrite) except EnvironmentError as error: - raise cdist.Error('Error creating directories for cdist object: %s: %s' % (self, error)) + raise cdist.Error(('Error creating directories for cdist object: ' + '%s: %s') % (self, error)) def requirements_unfinished(self, requirements): """Return state whether requirements are satisfied""" diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index d5b4ea3e..a548f365 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -24,6 +24,7 @@ import os import cdist + class NoSuchTypeError(cdist.Error): def __init__(self, name, type_path, type_absolute_path): self.name = name @@ -31,7 +32,8 @@ class NoSuchTypeError(cdist.Error): self.type_absolute_path = type_absolute_path def __str__(self): - return "Type '%s' does not exist at %s" % (self.type_path, self.type_absolute_path) + return "Type '%s' does not exist at %s" % ( + self.type_path, self.type_absolute_path) class CdistType(object): @@ -75,13 +77,13 @@ class CdistType(object): """Return a list of type names""" return os.listdir(base_path) - _instances = {} + def __new__(cls, *args, **kwargs): """only one instance of each named type may exist""" # name is second argument name = args[1] - if not name in cls._instances: + if name not in cls._instances: instance = super(CdistType, cls).__new__(cls) cls._instances[name] = instance # return instance so __init__ is called @@ -103,7 +105,8 @@ class CdistType(object): @property def is_install(self): - """Check whether a type is used for installation (if not: for configuration)""" + """Check whether a type is used for installation + (if not: for configuration)""" return os.path.isfile(os.path.join(self.absolute_path, "install")) @property @@ -111,7 +114,8 @@ class CdistType(object): """Return a list of available explorers""" if not self.__explorers: try: - self.__explorers = os.listdir(os.path.join(self.absolute_path, "explorer")) + self.__explorers = os.listdir(os.path.join(self.absolute_path, + "explorer")) except EnvironmentError: # error ignored self.__explorers = [] @@ -123,7 +127,9 @@ class CdistType(object): if not self.__required_parameters: parameters = [] try: - with open(os.path.join(self.absolute_path, "parameter", "required")) as fd: + with open(os.path.join(self.absolute_path, + "parameter", + "required")) as fd: for line in fd: parameters.append(line.strip()) except EnvironmentError: @@ -139,7 +145,9 @@ class CdistType(object): if not self.__required_multiple_parameters: parameters = [] try: - with open(os.path.join(self.absolute_path, "parameter", "required_multiple")) as fd: + with open(os.path.join(self.absolute_path, + "parameter", + "required_multiple")) as fd: for line in fd: parameters.append(line.strip()) except EnvironmentError: @@ -155,7 +163,9 @@ class CdistType(object): if not self.__optional_parameters: parameters = [] try: - with open(os.path.join(self.absolute_path, "parameter", "optional")) as fd: + with open(os.path.join(self.absolute_path, + "parameter", + "optional")) as fd: for line in fd: parameters.append(line.strip()) except EnvironmentError: @@ -171,7 +181,9 @@ class CdistType(object): if not self.__optional_multiple_parameters: parameters = [] try: - with open(os.path.join(self.absolute_path, "parameter", "optional_multiple")) as fd: + with open(os.path.join(self.absolute_path, + "parameter", + "optional_multiple")) as fd: for line in fd: parameters.append(line.strip()) except EnvironmentError: @@ -187,7 +199,9 @@ class CdistType(object): if not self.__boolean_parameters: parameters = [] try: - with open(os.path.join(self.absolute_path, "parameter", "boolean")) as fd: + with open(os.path.join(self.absolute_path, + "parameter", + "boolean")) as fd: for line in fd: parameters.append(line.strip()) except EnvironmentError: @@ -202,7 +216,9 @@ class CdistType(object): if not self.__parameter_defaults: defaults = {} try: - defaults_dir = os.path.join(self.absolute_path, "parameter", "default") + defaults_dir = os.path.join(self.absolute_path, + "parameter", + "default") for name in os.listdir(defaults_dir): try: with open(os.path.join(defaults_dir, name)) as fd: diff --git a/cdist/core/code.py b/cdist/core/code.py index 64517532..8e3cd63e 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -37,15 +37,17 @@ common: PATH: prepend directory with type emulator symlinks == local.bin_path __target_host: the target host we are working on __cdist_manifest: full qualified path of the manifest == script - __cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator - == local.type_path + __cdist_type_base_path: full qualified path to the directory where + types are defined for use in type emulator + == local.type_path gencode-local script: full qualified path to a types gencode-local env: __target_host: the target host we are working on - __global: full qualified path to the global output dir == local.out_path + __global: full qualified path to the global + output dir == local.out_path __object: full qualified path to the object's dir __object_id: the objects id __object_fq: full qualified object id, iow: $type.name + / + object_id @@ -59,7 +61,8 @@ gencode-remote env: __target_host: the target host we are working on - __global: full qualified path to the global output dir == local.out_path + __global: full qualified path to the global + output dir == local.out_path __object: full qualified path to the object's dir __object_id: the objects id __object_fq: full qualified object id, iow: $type.name + / + object_id @@ -98,7 +101,8 @@ class Code(object): def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type - script = os.path.join(self.local.type_path, getattr(cdist_type, 'gencode_%s_path' % which)) + script = os.path.join(self.local.type_path, + getattr(cdist_type, 'gencode_%s_path' % which)) if os.path.isfile(script): env = os.environ.copy() env.update(self.env) @@ -108,8 +112,9 @@ class Code(object): '__object_id': cdist_object.object_id, '__object_name': cdist_object.name, }) - message_prefix=cdist_object.name - return self.local.run_script(script, env=env, return_output=True, message_prefix=message_prefix) + message_prefix = cdist_object.name + return self.local.run_script(script, env=env, return_output=True, + message_prefix=message_prefix) def run_gencode_local(self, cdist_object): """Run the gencode-local script for the given cdist object.""" @@ -120,21 +125,26 @@ class Code(object): return self._run_gencode(cdist_object, 'remote') def transfer_code_remote(self, cdist_object): - """Transfer the code_remote script for the given object to the remote side.""" - source = os.path.join(self.local.object_path, cdist_object.code_remote_path) - destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path) + """Transfer the code_remote script for the given object to the + remote side.""" + source = os.path.join(self.local.object_path, + cdist_object.code_remote_path) + destination = os.path.join(self.remote.object_path, + cdist_object.code_remote_path) # FIXME: BUG: do not create destination, but top level of destination! self.remote.mkdir(destination) self.remote.transfer(source, destination) def _run_code(self, cdist_object, which, env=None): which_exec = getattr(self, which) - script = os.path.join(which_exec.object_path, getattr(cdist_object, 'code_%s_path' % which)) + script = os.path.join(which_exec.object_path, + getattr(cdist_object, 'code_%s_path' % which)) return which_exec.run_script(script, env=env) def run_code_local(self, cdist_object): """Run the code-local script for the given cdist object.""" - # Put some env vars, to allow read only access to the parameters over $__object + # Put some env vars, to allow read only access to the parameters + # over $__object env = os.environ.copy() env.update(self.env) env.update({ @@ -144,10 +154,13 @@ class Code(object): return self._run_code(cdist_object, 'local', env=env) def run_code_remote(self, cdist_object): - """Run the code-remote script for the given cdist object on the remote side.""" - # Put some env vars, to allow read only access to the parameters over $__object which is already on the remote side + """Run the code-remote script for the given cdist object on the + remote side.""" + # Put some env vars, to allow read only access to the parameters + # over $__object which is already on the remote side env = { - '__object': os.path.join(self.remote.object_path, cdist_object.path), + '__object': os.path.join(self.remote.object_path, + cdist_object.path), '__object_id': cdist_object.object_id, } return self._run_code(cdist_object, 'remote', env=env) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 41851bd6..345f45ff 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -31,7 +31,8 @@ common: runs only remotely, needs local and remote to construct paths env: - __explorer: full qualified path to other global explorers on remote side + __explorer: full qualified path to other global explorers on + remote side -> remote.global_explorer_path a global explorer is: @@ -52,7 +53,8 @@ type explorer is: __object: full qualified path to the object's remote dir __object_id: the objects id __object_fq: full qualified object id, iow: $type.name + / + object_id - __type_explorer: full qualified path to the other type explorers on remote side + __type_explorer: full qualified path to the other type explorers on + remote side creates: nothing, returns output @@ -76,7 +78,7 @@ class Explorer(object): } self._type_explorers_transferred = [] - ### global + # global def list_global_explorer_names(self): """Return a list of global explorer names.""" @@ -98,15 +100,17 @@ class Explorer(object): def transfer_global_explorers(self): """Transfer the global explorers to the remote side.""" self.remote.mkdir(self.remote.global_explorer_path) - self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path) - self.remote.run(["chmod", "0700", "%s/*" % (self.remote.global_explorer_path)]) + self.remote.transfer(self.local.global_explorer_path, + self.remote.global_explorer_path) + self.remote.run(["chmod", "0700", + "%s/*" % (self.remote.global_explorer_path)]) def run_global_explorer(self, explorer): """Run the given global explorer and return it's output.""" script = os.path.join(self.remote.global_explorer_path, explorer) return self.remote.run_script(script, env=self.env, return_output=True) - ### type + # type def list_type_explorer_names(self, cdist_type): """Return a list of explorer names for the given type.""" @@ -121,37 +125,48 @@ class Explorer(object): in the object. """ - self.log.debug("Transfering type explorers for type: %s", cdist_object.cdist_type) + self.log.debug("Transfering type explorers for type: %s", + cdist_object.cdist_type) self.transfer_type_explorers(cdist_object.cdist_type) - self.log.debug("Transfering object parameters for object: %s", cdist_object.name) + self.log.debug("Transfering object parameters for object: %s", + cdist_object.name) self.transfer_object_parameters(cdist_object) for explorer in self.list_type_explorer_names(cdist_object.cdist_type): output = self.run_type_explorer(explorer, cdist_object) - self.log.debug("Running type explorer '%s' for object '%s'", explorer, cdist_object.name) + self.log.debug("Running type explorer '%s' for object '%s'", + explorer, cdist_object.name) cdist_object.explorers[explorer] = output def run_type_explorer(self, explorer, cdist_object): - """Run the given type explorer for the given object and return it's output.""" + """Run the given type explorer for the given object and return + it's output.""" cdist_type = cdist_object.cdist_type env = self.env.copy() env.update({ - '__object': os.path.join(self.remote.object_path, cdist_object.path), + '__object': os.path.join(self.remote.object_path, + cdist_object.path), '__object_id': cdist_object.object_id, '__object_name': cdist_object.name, '__object_fq': cdist_object.path, - '__type_explorer': os.path.join(self.remote.type_path, cdist_type.explorer_path) + '__type_explorer': os.path.join(self.remote.type_path, + cdist_type.explorer_path) }) - script = os.path.join(self.remote.type_path, cdist_type.explorer_path, explorer) + script = os.path.join(self.remote.type_path, cdist_type.explorer_path, + explorer) return self.remote.run_script(script, env=env, return_output=True) def transfer_type_explorers(self, cdist_type): - """Transfer the type explorers for the given type to the remote side.""" + """Transfer the type explorers for the given type to the + remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.debug("Skipping retransfer of type explorers for: %s", cdist_type) + self.log.debug("Skipping retransfer of type explorers for: %s", + cdist_type) else: - source = os.path.join(self.local.type_path, cdist_type.explorer_path) - destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) + source = os.path.join(self.local.type_path, + cdist_type.explorer_path) + destination = os.path.join(self.remote.type_path, + cdist_type.explorer_path) self.remote.mkdir(destination) self.remote.transfer(source, destination) self.remote.run(["chmod", "0700", "%s/*" % (destination)]) @@ -160,7 +175,9 @@ class Explorer(object): def transfer_object_parameters(self, cdist_object): """Transfer the parameters for the given object to the remote side.""" if cdist_object.parameters: - source = os.path.join(self.local.object_path, cdist_object.parameter_path) - destination = os.path.join(self.remote.object_path, cdist_object.parameter_path) + source = os.path.join(self.local.object_path, + cdist_object.parameter_path) + destination = os.path.join(self.remote.object_path, + cdist_object.parameter_path) self.remote.mkdir(destination) self.remote.transfer(source, destination) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 3b71a215..ac322101 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -32,9 +32,11 @@ common: env: PATH: prepend directory with type emulator symlinks == local.bin_path __target_host: the target host we are working on - __global: full qualified path to the global output dir == local.out_path + __global: full qualified path to the global + output dir == local.out_path __cdist_manifest: full qualified path of the manifest == script - __cdist_type_base_path: full qualified path to the directory where types are defined for use in type emulator + __cdist_type_base_path: full qualified path to the directory where + types are defined for use in type emulator == local.type_path __files: full qualified path to the files dir @@ -58,6 +60,7 @@ type manifeste is: creates: new objects through type emulator ''' + class NoInitialManifestError(cdist.Error): """ Display missing initial manifest: @@ -72,7 +75,9 @@ class NoInitialManifestError(cdist.Error): if user_supplied: if os.path.islink(manifest_path): - self.message = "%s: %s -> %s" % (msg_header, manifest_path, os.path.realpath(manifest_path)) + self.message = "%s: %s -> %s" % ( + msg_header, manifest_path, + os.path.realpath(manifest_path)) else: self.message = "%s: %s" % (msg_header, manifest_path) else: @@ -94,14 +99,15 @@ class Manifest(object): self.env = { 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), - '__cdist_type_base_path': self.local.type_path, # for use in type emulator + # for use in type emulator + '__cdist_type_base_path': self.local.type_path, '__global': self.local.base_path, '__target_host': self.target_host, '__files': self.local.files_path, } - if self.log.getEffectiveLevel() == logging.DEBUG: - self.env.update({'__cdist_debug': "yes" }) + if self.log.getEffectiveLevel() == logging.DEBUG: + self.env.update({'__cdist_debug': "yes"}) def env_initial_manifest(self, initial_manifest): env = os.environ.copy() @@ -124,11 +130,14 @@ class Manifest(object): if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) - message_prefix="initialmanifest" - self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), message_prefix=message_prefix) + message_prefix = "initialmanifest" + self.local.run_script(initial_manifest, + env=self.env_initial_manifest(initial_manifest), + message_prefix=message_prefix) def env_type_manifest(self, cdist_object): - type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path) + type_manifest = os.path.join(self.local.type_path, + cdist_object.cdist_type.manifest_path) env = os.environ.copy() env.update(self.env) env.update({ @@ -143,7 +152,10 @@ class Manifest(object): return env def run_type_manifest(self, cdist_object): - type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path) + type_manifest = os.path.join(self.local.type_path, + cdist_object.cdist_type.manifest_path) message_prefix = cdist_object.name if os.path.isfile(type_manifest): - self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix) + self.local.run_script(type_manifest, + env=self.env_type_manifest(cdist_object), + message_prefix=message_prefix) diff --git a/cdist/emulator.py b/cdist/emulator.py index 0e4a41c4..58ab7389 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -29,10 +29,12 @@ import sys import cdist from cdist import core + class MissingRequiredEnvironmentVariableError(cdist.Error): def __init__(self, name): self.name = name - self.message = "Emulator requires the environment variable %s to be setup" % self.name + self.message = ("Emulator requires the environment variable %s to be " + "setup" % self.name) def __str__(self): return self.message @@ -41,7 +43,7 @@ class MissingRequiredEnvironmentVariableError(cdist.Error): class DefaultList(list): """Helper class to allow default values for optional_multiple parameters. - @see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ + @see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ """ def __copy__(self): return [] @@ -54,20 +56,20 @@ class DefaultList(list): class Emulator(object): def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ): - self.argv = argv - self.stdin = stdin - self.env = env + self.argv = argv + self.stdin = stdin + self.env = env - self.object_id = '' + self.object_id = '' try: - self.global_path = self.env['__global'] - self.target_host = self.env['__target_host'] + self.global_path = self.env['__global'] + self.target_host = self.env['__target_host'] # Internal variables - self.object_source = self.env['__cdist_manifest'] + self.object_source = self.env['__cdist_manifest'] self.type_base_path = self.env['__cdist_type_base_path'] - self.object_marker = self.env['__cdist_object_marker'] + self.object_marker = self.env['__cdist_object_marker'] except KeyError as e: raise MissingRequiredEnvironmentVariableError(e.args[0]) @@ -75,8 +77,8 @@ class Emulator(object): self.object_base_path = os.path.join(self.global_path, "object") self.typeorder_path = os.path.join(self.global_path, "typeorder") - self.type_name = os.path.basename(argv[0]) - self.cdist_type = core.CdistType(self.type_base_path, self.type_name) + self.type_name = os.path.basename(argv[0]) + self.cdist_type = core.CdistType(self.type_base_path, self.type_name) self.__init_log() @@ -88,7 +90,8 @@ class Emulator(object): self.save_stdin() self.record_requirements() self.record_auto_requirements() - self.log.debug("Finished %s %s" % (self.cdist_object.path, self.parameters)) + self.log.debug("Finished %s %s" % ( + self.cdist_object.path, self.parameters)) def __init_log(self): """Setup logging facility""" @@ -98,30 +101,38 @@ class Emulator(object): else: logging.root.setLevel(logging.INFO) - self.log = logging.getLogger(self.target_host) + self.log = logging.getLogger(self.target_host) def commandline(self): """Parse command line""" - parser = argparse.ArgumentParser(add_help=False, argument_default=argparse.SUPPRESS) + parser = argparse.ArgumentParser(add_help=False, + argument_default=argparse.SUPPRESS) for parameter in self.cdist_type.required_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='store', required=True) + parser.add_argument(argument, dest=parameter, action='store', + required=True) for parameter in self.cdist_type.required_multiple_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='append', required=True) + parser.add_argument(argument, dest=parameter, action='append', + required=True) for parameter in self.cdist_type.optional_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='store', required=False, - default=self.cdist_type.parameter_defaults.get(parameter, None)) + default = self.cdist_type.parameter_defaults.get(parameter, None) + parser.add_argument(argument, dest=parameter, action='store', + required=False, default=default) for parameter in self.cdist_type.optional_multiple_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='append', required=False, - default=DefaultList.create(self.cdist_type.parameter_defaults.get(parameter, None))) + default = DefaultList.create( + self.cdist_type.parameter_defaults.get( + parameter, None)) + parser.add_argument(argument, dest=parameter, action='append', + required=False, default=default) for parameter in self.cdist_type.boolean_parameters: argument = "--" + parameter - parser.add_argument(argument, dest=parameter, action='store_const', const='') + parser.add_argument(argument, dest=parameter, + action='store_const', const='') # If not singleton support one positional parameter if not self.cdist_type.is_singleton: @@ -140,30 +151,33 @@ class Emulator(object): del self.args.object_id # Instantiate the cdist object we are defining - self.cdist_object = core.CdistObject(self.cdist_type, - self.object_base_path, self.object_marker, self.object_id) + self.cdist_object = core.CdistObject( + self.cdist_type, self.object_base_path, self.object_marker, + self.object_id) # Create object with given parameters self.parameters = {} - for key,value in vars(self.args).items(): + for key, value in vars(self.args).items(): if value is not None: self.parameters[key] = value - if self.cdist_object.exists and not 'CDIST_OVERRIDE' in self.env: + if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: # make existing requirements a set, so we can compare it # later with new requirements if self.cdist_object.parameters != self.parameters: errmsg = ("Object %s already exists with conflicting " - "parameters:\n%s: %s\n%s: %s" % (self.cdist_object.name, - " ".join(self.cdist_object.source), - self.cdist_object.parameters, - self.object_source, - self.parameters)) + "parameters:\n%s: %s\n%s: %s" % ( + self.cdist_object.name, + " ".join(self.cdist_object.source), + self.cdist_object.parameters, + self.object_source, + self.parameters)) self.log.error(errmsg) raise cdist.Error(errmsg) else: if self.cdist_object.exists: - self.log.debug('Object %s override forced with CDIST_OVERRIDE',self.cdist_object.name) + self.log.debug(('Object %s override forced with ' + 'CDIST_OVERRIDE'), self.cdist_object.name) self.cdist_object.create(True) else: self.cdist_object.create() @@ -176,6 +190,7 @@ class Emulator(object): self.cdist_object.source.append(self.object_source) chunk_size = 65536 + def _read_stdin(self): return self.stdin.read(self.chunk_size) @@ -197,7 +212,6 @@ class Emulator(object): except EnvironmentError as e: raise cdist.Error('Failed to read from stdin: %s' % e) - def record_requirement(self, requirement): """record requirement and return recorded requirement""" @@ -206,13 +220,15 @@ class Emulator(object): cdist_object = self.cdist_object.object_from_name(requirement) except core.cdist_type.NoSuchTypeError as e: self.log.error(("%s requires object %s, but type %s does not" - " exist. Defined at %s" % (self.cdist_object.name, - requirement, e.name, self.object_source))) + " exist. Defined at %s" % ( + self.cdist_object.name, + requirement, e.name, self.object_source))) raise except core.cdist_object.MissingObjectIdError as e: self.log.error(("%s requires object %s without object id." - " Defined at %s" % (self.cdist_object.name, requirement, - self.object_source))) + " Defined at %s" % (self.cdist_object.name, + requirement, + self.object_source))) raise self.log.debug("Recording requirement: %s", requirement) @@ -224,12 +240,13 @@ class Emulator(object): return cdist_object.name - def record_requirements(self): """record requirements""" - # Inject the predecessor, but not if its an override (this would leed to an circular dependency) - if "CDIST_ORDER_DEPENDENCY" in self.env and not 'CDIST_OVERRIDE' in self.env: + # Inject the predecessor, but not if its an override + # (this would leed to an circular dependency) + if ("CDIST_ORDER_DEPENDENCY" in self.env and + 'CDIST_OVERRIDE' not in self.env): # load object name created bevor this one from typeorder file ... with open(self.typeorder_path, 'r') as typecreationfile: typecreationorder = typecreationfile.readlines() @@ -240,9 +257,12 @@ class Emulator(object): self.env['require'] += " " + lastcreatedtype else: self.env['require'] = lastcreatedtype - self.log.debug("Injecting require for CDIST_ORDER_DEPENDENCY: %s for %s", lastcreatedtype, self.cdist_object.name) + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, self.cdist_object.name) except IndexError: - # if no second last line, we are on the first type, so do not set a requirement + # if no second last line, we are on the first type, + # so do not set a requirement pass if "require" in self.env: @@ -250,22 +270,25 @@ class Emulator(object): self.log.debug("reqs = " + requirements) for requirement in requirements.split(" "): # Ignore empty fields - probably the only field anyway - if len(requirement) == 0: continue + if len(requirement) == 0: + continue self.record_requirement(requirement) - def record_auto_requirements(self): - """An object shall automatically depend on all objects that it defined in it's type manifest. + """An object shall automatically depend on all objects that it + defined in it's type manifest. """ - # __object_name is the name of the object whose type manifest is currently executed + # __object_name is the name of the object whose type manifest is + # currently executed __object_name = self.env.get('__object_name', None) if __object_name: # The object whose type manifest is currently run parent = self.cdist_object.object_from_name(__object_name) # The object currently being defined current_object = self.cdist_object - # As parent defined current_object it shall automatically depend on it. + # As parent defined current_object it shall automatically + # depend on it. # But only if the user hasn't said otherwise. # Must prevent circular dependencies. - if not parent.name in current_object.requirements: + if parent.name not in current_object.requirements: parent.autorequire.append(current_object.name) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 5a0a8644..48067543 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -35,7 +35,8 @@ import cdist.message from cdist import core import cdist.exec.util as exec_util -CONF_SUBDIRS_LINKED = [ "explorer", "files", "manifest", "type" ] +CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] + class Local(object): """Execute commands locally. @@ -82,7 +83,8 @@ class Local(object): @property def dist_conf_dir(self): - return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")) + return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), + "conf")) @property def home_dir(self): @@ -109,7 +111,8 @@ class Local(object): # Depending on out_path self.bin_path = os.path.join(self.base_path, "bin") self.conf_path = os.path.join(self.base_path, "conf") - self.global_explorer_out_path = os.path.join(self.base_path, "explorer") + self.global_explorer_out_path = os.path.join(self.base_path, + "explorer") self.object_path = os.path.join(self.base_path, "object") self.messages_path = os.path.join(self.base_path, "messages") self.files_path = os.path.join(self.conf_path, "files") @@ -118,7 +121,7 @@ class Local(object): self.global_explorer_path = os.path.join(self.conf_path, "explorer") self.manifest_path = os.path.join(self.conf_path, "manifest") self.initial_manifest = (self.custom_initial_manifest or - os.path.join(self.manifest_path, "init")) + os.path.join(self.manifest_path, "init")) self.type_path = os.path.join(self.conf_path, "type") @@ -164,8 +167,8 @@ class Local(object): with open(self.object_marker_file, 'w') as fd: fd.write("%s\n" % self.object_marker_name) - self.log.debug("Object marker %s saved in %s" % (self.object_marker_name, self.object_marker_file)) - + self.log.debug("Object marker %s saved in %s" % ( + self.object_marker_name, self.object_marker_file)) def _init_cache_dir(self, cache_dir): if cache_dir: @@ -174,7 +177,8 @@ class Local(object): if self.home_dir: self.cache_path = os.path.join(self.home_dir, "cache") else: - raise cdist.Error("No homedir setup and no cache dir location given") + raise cdist.Error( + "No homedir setup and no cache dir location given") def rmdir(self, path): """Remove directory on the local side.""" @@ -192,7 +196,8 @@ class Local(object): """ self.log.debug("Local run: %s", command) - assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command + assert isinstance(command, (list, tuple)), ( + "list or tuple argument expected, got: %s" % command) if env is None: env = os.environ.copy() @@ -220,16 +225,17 @@ class Local(object): if message_prefix: message.merge_messages() - def run_script(self, script, env=None, return_output=False, message_prefix=None): + def run_script(self, script, env=None, return_output=False, + message_prefix=None): """Run the given script with the given environment. Return the output as a string. """ - command = [ os.environ.get('CDIST_LOCAL_SHELL',"/bin/sh") , "-e"] + command = [os.environ.get('CDIST_LOCAL_SHELL', "/bin/sh"), "-e"] command.append(script) - return self.run(command=command, env=env, return_output=return_output, message_prefix=message_prefix) - + return self.run(command=command, env=env, return_output=return_output, + message_prefix=message_prefix) def save_cache(self): destination = os.path.join(self.cache_path, self.hostdir) @@ -239,7 +245,8 @@ class Local(object): if os.path.exists(destination): shutil.rmtree(destination) except PermissionError as e: - raise cdist.Error("Cannot delete old cache %s: %s" % (destination, e)) + raise cdist.Error( + "Cannot delete old cache %s: %s" % (destination, e)) shutil.move(self.base_path, destination) @@ -265,18 +272,21 @@ class Local(object): for entry in os.listdir(current_dir): rel_entry_path = os.path.join(sub_dir, entry) - src = os.path.abspath(os.path.join(conf_dir, sub_dir, entry)) + src = os.path.abspath(os.path.join(conf_dir, + sub_dir, + entry)) dst = os.path.join(self.conf_path, sub_dir, entry) # Already exists? remove and link if os.path.exists(dst): os.unlink(dst) - + self.log.debug("Linking %s to %s ..." % (src, dst)) try: os.symlink(src, dst) except OSError as e: - raise cdist.Error("Linking %s %s to %s failed: %s" % (sub_dir, src, dst, e.__str__())) + raise cdist.Error("Linking %s %s to %s failed: %s" % ( + sub_dir, src, dst, e.__str__())) def _link_types_for_emulator(self): """Link emulator to types""" @@ -288,4 +298,6 @@ class Local(object): try: os.symlink(src, dst) except OSError as e: - raise cdist.Error("Linking emulator from %s to %s failed: %s" % (src, dst, e.__str__())) + raise cdist.Error( + "Linking emulator from %s to %s failed: %s" % ( + src, dst, e.__str__())) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 0331f402..ff388e1d 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -30,6 +30,7 @@ import cdist.exec.util as exec_util import cdist + class DecodeError(cdist.Error): def __init__(self, command): self.command = command @@ -75,7 +76,6 @@ class Remote(object): os.environ['__remote_copy'] = self._copy os.environ['__remote_exec'] = self._exec - def create_files_dirs(self): self.rmdir(self.base_path) self.mkdir(self.base_path) @@ -101,11 +101,13 @@ class Remote(object): for f in glob.glob1(source, '*'): command = self._copy.split() path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format(self.target_host, destination)]) + command.extend([path, '{0}:{1}'.format( + self.target_host, destination)]) self._run_command(command) else: command = self._copy.split() - command.extend([source, '{0}:{1}'.format(self.target_host, destination)]) + command.extend([source, '{0}:{1}'.format( + self.target_host, destination)]) self._run_command(command) def run_script(self, script, env=None, return_output=False): @@ -114,7 +116,7 @@ class Remote(object): """ - command = [ os.environ.get('CDIST_REMOTE_SHELL',"/bin/sh") , "-e"] + command = [os.environ.get('CDIST_REMOTE_SHELL', "/bin/sh"), "-e"] command.append(script) return self.run(command, env, return_output) @@ -148,8 +150,8 @@ class Remote(object): # /bin/csh will execute this script in the right way. if env: remote_env = [" export %s=%s;" % item for item in env.items()] - string_cmd = ("/bin/sh -c '" + " ".join(remote_env) - + " ".join(command) + "'") + string_cmd = ("/bin/sh -c '" + " ".join(remote_env) + + " ".join(command) + "'") cmd.append(string_cmd) else: cmd.extend(command) @@ -160,7 +162,8 @@ class Remote(object): Return the output as a string. """ - assert isinstance(command, (list, tuple)), "list or tuple argument expected, got: %s" % command + assert isinstance(command, (list, tuple)), ( + "list or tuple argument expected, got: %s" % command) # export target_host for use in __remote_{exec,copy} scripts os_environ = os.environ.copy() diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 15430f13..a1bca296 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -27,6 +27,7 @@ import cdist STDERR_UNSUPPORTED = 'Not supported in this python version' + def call_get_output(command, env=None): """Run the given command with the given environment. Return the tuple of stdout and stderr output as a byte strings. @@ -40,14 +41,16 @@ def call_get_output(command, env=None): else: return (call_get_stdout(command, env), STDERR_UNSUPPORTED) + def handle_called_process_error(err, command): if sys.version_info >= (3, 5): errout = err.stderr else: errout = STDERR_UNSUPPORTED - raise cdist.Error("Command failed: " + " ".join(command) - + " with returncode: {} and stdout: {}, stderr: {}".format( - err.returncode, err.output, errout)) + raise cdist.Error("Command failed: " + " ".join(command) + + " with returncode: {} and stdout: {}, stderr: {}".format( + err.returncode, err.output, errout)) + def call_get_stdout(command, env=None): """Run the given command with the given environment. @@ -63,6 +66,7 @@ def call_get_stdout(command, env=None): return output + def call_get_out_err(command, env=None): """Run the given command with the given environment. Return the tuple of stdout and stderr output as a byte strings. @@ -72,7 +76,7 @@ def call_get_out_err(command, env=None): with TemporaryFile() as fout, TemporaryFile() as ferr: subprocess.check_call(command, env=env, - stdout=fout, stderr=ferr) + stdout=fout, stderr=ferr) fout.seek(0) ferr.seek(0) output = (fout.read(), ferr.read()) diff --git a/cdist/log.py b/cdist/log.py index 8c3aac79..2341c282 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -22,6 +22,7 @@ import logging + class Log(logging.Logger): def __init__(self, name): diff --git a/cdist/message.py b/cdist/message.py index f9b0c313..98a6e8cf 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -37,8 +37,9 @@ class Message(object): self.prefix = prefix self.global_messages = messages - in_fd, self.messages_in = tempfile.mkstemp(suffix='.cdist_message_in') - out_fd, self.messages_out = tempfile.mkstemp(suffix='.cdist_message_out') + in_fd, self.messages_in = tempfile.mkstemp(suffix='.cdist_message_in') + out_fd, self.messages_out = tempfile.mkstemp( + suffix='.cdist_message_out') os.close(in_fd) os.close(out_fd) @@ -48,7 +49,7 @@ class Message(object): @property def env(self): env = {} - env['__messages_in'] = self.messages_in + env['__messages_in'] = self.messages_in env['__messages_out'] = self.messages_out return env diff --git a/cdist/shell.py b/cdist/shell.py index 83b2acf7..60faa2f8 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -31,8 +31,9 @@ import cdist.config log = logging.getLogger(__name__) + class Shell(object): - + def __init__(self, shell=None): self.shell = shell @@ -45,16 +46,17 @@ class Shell(object): """Select shell to execute, if not specified by user""" if not self.shell: - self.shell = os.environ.get('SHELL',"/bin/sh") + self.shell = os.environ.get('SHELL', "/bin/sh") def _init_files_dirs(self): self.local.create_files_dirs() def _init_environment(self): self.env = os.environ.copy() - additional_env = { + additional_env = { 'PATH': "%s:%s" % (self.local.bin_path, os.environ['PATH']), - '__cdist_type_base_path': self.local.type_path, # for use in type emulator + # for use in type emulator + '__cdist_type_base_path': self.local.type_path, '__cdist_manifest': "cdist shell", '__global': self.local.base_path, '__target_host': self.target_host, diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py index 433f6a1b..97b41f03 100644 --- a/cdist/sphinxext/manpage.py +++ b/cdist/sphinxext/manpage.py @@ -14,12 +14,13 @@ from sphinx import addnodes but leaves everything to actual reStructuredText file content. """ + class ManualPageTranslator(sphinx.writers.manpage.ManualPageTranslator): - def header(self): - tmpl = (".TH \"%(title_upper)s\" \"%(manual_section)s\"" - " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n") - return tmpl % self._docinfo + def header(self): + tmpl = (".TH \"%(title_upper)s\" \"%(manual_section)s\"" + " \"%(date)s\" \"%(version)s\" \"%(manual_group)s\"\n") + return tmpl % self._docinfo class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): @@ -76,5 +77,6 @@ class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): docwriter.write(largetree, destination) self.info() + def setup(app): app.add_builder(ManualPageBuilder) diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py index ab767699..d9630f53 100644 --- a/cdist/test/__init__.py +++ b/cdist/test/__init__.py @@ -28,7 +28,9 @@ cdist_base_path = os.path.abspath( cdist_exec_path = os.path.join(cdist_base_path, "scripts/cdist") -global_fixtures_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "fixtures")) +global_fixtures_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), + "fixtures")) + class CdistTestCase(unittest.TestCase): diff --git a/cdist/test/banner/__init__.py b/cdist/test/banner/__init__.py index 4b0ab6ac..957b7b12 100644 --- a/cdist/test/banner/__init__.py +++ b/cdist/test/banner/__init__.py @@ -27,6 +27,7 @@ import unittest import cdist import cdist.banner + class Banner(unittest.TestCase): def setUp(self): self.banner = cdist.BANNER + "\n" @@ -38,5 +39,5 @@ class Banner(unittest.TestCase): sys.stdout = output cdist.banner.banner(None) - + self.assertEqual(output.getvalue(), self.banner) diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index 9c075acb..d03c0642 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -57,29 +57,37 @@ class ObjectClassTestCase(test.CdistTestCase): self.expected_objects = [] for cdist_object_name in expected_object_names: cdist_type, cdist_object_id = cdist_object_name.split("/", 1) - cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), self.object_base_path, - OBJECT_MARKER_NAME, cdist_object_id) + cdist_object = core.CdistObject(core.CdistType(type_base_path, + cdist_type), + self.object_base_path, + OBJECT_MARKER_NAME, + cdist_object_id) cdist_object.create() self.expected_objects.append(cdist_object) - + def tearDown(self): shutil.rmtree(self.tempdir) def test_list_object_names(self): - found_object_names = sorted(list(core.CdistObject.list_object_names(self.object_base_path, OBJECT_MARKER_NAME))) - self.assertEqual(found_object_names, expected_object_names) + found_object_names = sorted(list(core.CdistObject.list_object_names( + self.object_base_path, OBJECT_MARKER_NAME))) + self.assertEqual(found_object_names, expected_object_names) def test_list_type_names(self): - type_names = list(cdist.core.CdistObject.list_type_names(self.object_base_path)) - self.assertEqual(sorted(type_names), ['__first', '__second', '__third']) + type_names = list(cdist.core.CdistObject.list_type_names( + self.object_base_path)) + self.assertEqual(sorted(type_names), + ['__first', '__second', '__third']) def test_list_objects(self): - found_objects = sorted(list(core.CdistObject.list_objects(self.object_base_path, type_base_path, OBJECT_MARKER_NAME))) + found_objects = sorted(list(core.CdistObject.list_objects( + self.object_base_path, type_base_path, OBJECT_MARKER_NAME))) self.assertEqual(found_objects, self.expected_objects) def test_create_singleton(self): """Check whether creating an object without id (singleton) works""" - singleton = self.expected_objects[0].object_from_name("__test_singleton") + singleton = self.expected_objects[0].object_from_name( + "__test_singleton") # came here - everything fine def test_create_singleton_not_singleton_type(self): @@ -88,6 +96,7 @@ class ObjectClassTestCase(test.CdistTestCase): with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): self.expected_objects[0].object_from_name("__first") + class ObjectIdTestCase(test.CdistTestCase): def setUp(self): @@ -97,11 +106,14 @@ class ObjectIdTestCase(test.CdistTestCase): self.expected_objects = [] for cdist_object_name in expected_object_names: cdist_type, cdist_object_id = cdist_object_name.split("/", 1) - cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), self.object_base_path, - OBJECT_MARKER_NAME, cdist_object_id) + cdist_object = core.CdistObject(core.CdistType(type_base_path, + cdist_type), + self.object_base_path, + OBJECT_MARKER_NAME, + cdist_object_id) cdist_object.create() self.expected_objects.append(cdist_object) - + def tearDown(self): shutil.rmtree(self.tempdir) @@ -109,31 +121,39 @@ class ObjectIdTestCase(test.CdistTestCase): cdist_type = core.CdistType(type_base_path, '__third') illegal_object_id = '/object_id//may/not/contain/double/slash' with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, + OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME + illegal_object_id = ( + 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME) with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, + OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker_string(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME - core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) + illegal_object_id = ( + 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME) + core.CdistObject(cdist_type, self.object_base_path, + OBJECT_MARKER_NAME, illegal_object_id) # if we get here, the test passed def test_object_id_contains_only_dot(self): cdist_type = core.CdistType(type_base_path, '__third') illegal_object_id = '.' with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, + OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_on_singleton_type(self): cdist_type = core.CdistType(type_base_path, '__test_singleton') illegal_object_id = 'object_id' with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, + OBJECT_MARKER_NAME, illegal_object_id) + class ObjectTestCase(test.CdistTestCase): @@ -142,13 +162,14 @@ class ObjectTestCase(test.CdistTestCase): self.object_base_path = self.tempdir self.cdist_type = core.CdistType(type_base_path, '__third') - self.cdist_object = core.CdistObject(self.cdist_type, self.object_base_path, OBJECT_MARKER_NAME, 'moon') + self.cdist_object = core.CdistObject(self.cdist_type, + self.object_base_path, + OBJECT_MARKER_NAME, 'moon') self.cdist_object.create() self.cdist_object.parameters['planet'] = 'Saturn' self.cdist_object.parameters['name'] = 'Prometheus' - def tearDown(self): self.cdist_object.prepared = False self.cdist_object.ran = False @@ -166,22 +187,29 @@ class ObjectTestCase(test.CdistTestCase): self.assertEqual(self.cdist_object.object_id, 'moon') def test_path(self): - self.assertEqual(self.cdist_object.path, "__third/moon/%s" % OBJECT_MARKER_NAME) + self.assertEqual(self.cdist_object.path, + "__third/moon/%s" % OBJECT_MARKER_NAME) def test_absolute_path(self): - self.assertEqual(self.cdist_object.absolute_path, os.path.join(self.object_base_path, "__third/moon/%s" % OBJECT_MARKER_NAME)) + self.assertEqual(self.cdist_object.absolute_path, + os.path.join(self.object_base_path, + "__third/moon/%s" % OBJECT_MARKER_NAME)) def test_code_local_path(self): - self.assertEqual(self.cdist_object.code_local_path, "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) + self.assertEqual(self.cdist_object.code_local_path, + "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) def test_code_remote_path(self): - self.assertEqual(self.cdist_object.code_remote_path, "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) + self.assertEqual(self.cdist_object.code_remote_path, + "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) def test_parameter_path(self): - self.assertEqual(self.cdist_object.parameter_path, "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) + self.assertEqual(self.cdist_object.parameter_path, + "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) def test_explorer_path(self): - self.assertEqual(self.cdist_object.explorer_path, "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) + self.assertEqual(self.cdist_object.explorer_path, + "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) def test_parameters(self): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} @@ -190,32 +218,39 @@ class ObjectTestCase(test.CdistTestCase): def test_explorers(self): self.assertEqual(self.cdist_object.explorers, {}) - # FIXME: actually testing fsproperty.DirectoryDictProperty here, move to their own test case + # FIXME: actually testing fsproperty.DirectoryDictProperty here, + # move to their own test case def test_explorers_assign_dict(self): expected = {'first': 'foo', 'second': 'bar'} # when set, written to file self.cdist_object.explorers = expected - object_explorer_path = os.path.join(self.cdist_object.base_path, self.cdist_object.explorer_path) + object_explorer_path = os.path.join(self.cdist_object.base_path, + self.cdist_object.explorer_path) self.assertTrue(os.path.isdir(object_explorer_path)) # when accessed, read from file self.assertEqual(self.cdist_object.explorers, expected) # remove dynamically created folder self.cdist_object.explorers = {} - os.rmdir(os.path.join(self.cdist_object.base_path, self.cdist_object.explorer_path)) + os.rmdir(os.path.join(self.cdist_object.base_path, + self.cdist_object.explorer_path)) - # FIXME: actually testing fsproperty.DirectoryDictProperty here, move to their own test case + # FIXME: actually testing fsproperty.DirectoryDictProperty here, + # move to their own test case def test_explorers_assign_key_value(self): expected = {'first': 'foo', 'second': 'bar'} - object_explorer_path = os.path.join(self.cdist_object.base_path, self.cdist_object.explorer_path) - for key,value in expected.items(): + object_explorer_path = os.path.join(self.cdist_object.base_path, + self.cdist_object.explorer_path) + for key, value in expected.items(): # when set, written to file self.cdist_object.explorers[key] = value - self.assertTrue(os.path.isfile(os.path.join(object_explorer_path, key))) + self.assertTrue(os.path.isfile(os.path.join(object_explorer_path, + key))) # when accessed, read from file self.assertEqual(self.cdist_object.explorers, expected) # remove dynamically created folder self.cdist_object.explorers = {} - os.rmdir(os.path.join(self.cdist_object.base_path, self.cdist_object.explorer_path)) + os.rmdir(os.path.join(self.cdist_object.base_path, + self.cdist_object.explorer_path)) def test_requirements(self): expected = [] @@ -226,15 +261,18 @@ class ObjectTestCase(test.CdistTestCase): def test_state_prepared(self): self.cdist_object.state = core.CdistObject.STATE_PREPARED - self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_PREPARED) + self.assertEqual(self.cdist_object.state, + core.CdistObject.STATE_PREPARED) def test_state_running(self): self.cdist_object.state = core.CdistObject.STATE_RUNNING - self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_RUNNING) + self.assertEqual(self.cdist_object.state, + core.CdistObject.STATE_RUNNING) def test_state_done(self): self.cdist_object.state = core.CdistObject.STATE_DONE - self.assertEqual(self.cdist_object.state, core.CdistObject.STATE_DONE) + self.assertEqual(self.cdist_object.state, + core.CdistObject.STATE_DONE) def test_source(self): self.assertEqual(list(self.cdist_object.source), []) diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index ccbf6742..feb5fa15 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -34,7 +34,8 @@ class TypeTestCase(test.CdistTestCase): def test_list_type_names(self): base_path = op.join(fixtures, 'list_types') type_names = core.CdistType.list_type_names(base_path) - self.assertEqual(sorted(type_names), ['__first', '__second', '__third']) + self.assertEqual(sorted(type_names), + ['__first', '__second', '__third']) def test_list_types(self): base_path = op.join(fixtures, 'list_types') @@ -54,7 +55,8 @@ class TypeTestCase(test.CdistTestCase): def test_nonexistent_type(self): base_path = fixtures - self.assertRaises(core.NoSuchTypeError, core.CdistType, base_path, '__i-dont-exist') + self.assertRaises(core.NoSuchTypeError, core.CdistType, base_path, + '__i-dont-exist') def test_name(self): base_path = fixtures @@ -74,27 +76,32 @@ class TypeTestCase(test.CdistTestCase): def test_absolute_path(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__name_path') - self.assertEqual(cdist_type.absolute_path, os.path.join(base_path, '__name_path')) + self.assertEqual(cdist_type.absolute_path, + os.path.join(base_path, '__name_path')) def test_manifest_path(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__name_path') - self.assertEqual(cdist_type.manifest_path, os.path.join('__name_path', 'manifest')) + self.assertEqual(cdist_type.manifest_path, + os.path.join('__name_path', 'manifest')) def test_explorer_path(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__name_path') - self.assertEqual(cdist_type.explorer_path, os.path.join('__name_path', 'explorer')) + self.assertEqual(cdist_type.explorer_path, + os.path.join('__name_path', 'explorer')) def test_gencode_local_path(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__name_path') - self.assertEqual(cdist_type.gencode_local_path, os.path.join('__name_path', 'gencode-local')) + self.assertEqual(cdist_type.gencode_local_path, + os.path.join('__name_path', 'gencode-local')) def test_gencode_remote_path(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__name_path') - self.assertEqual(cdist_type.gencode_remote_path, os.path.join('__name_path', 'gencode-remote')) + self.assertEqual(cdist_type.gencode_remote_path, + os.path.join('__name_path', 'gencode-remote')) def test_singleton_is_singleton(self): base_path = fixtures @@ -119,17 +126,20 @@ class TypeTestCase(test.CdistTestCase): def test_with_required_parameters(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__with_required_parameters') - self.assertEqual(cdist_type.required_parameters, ['required1', 'required2']) + self.assertEqual(cdist_type.required_parameters, + ['required1', 'required2']) def test_without_required_parameters(self): base_path = fixtures - cdist_type = core.CdistType(base_path, '__without_required_parameters') + cdist_type = core.CdistType(base_path, + '__without_required_parameters') self.assertEqual(cdist_type.required_parameters, []) def test_with_optional_parameters(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__with_optional_parameters') - self.assertEqual(cdist_type.optional_parameters, ['optional1', 'optional2']) + self.assertEqual(cdist_type.optional_parameters, + ['optional1', 'optional2']) def test_without_optional_parameters(self): base_path = fixtures @@ -139,7 +149,8 @@ class TypeTestCase(test.CdistTestCase): def test_with_boolean_parameters(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__with_boolean_parameters') - self.assertEqual(cdist_type.boolean_parameters, ['boolean1', 'boolean2']) + self.assertEqual(cdist_type.boolean_parameters, + ['boolean1', 'boolean2']) def test_without_boolean_parameters(self): base_path = fixtures @@ -158,5 +169,4 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__directory_in_default') self.assertEqual( list(sorted(cdist_type.parameter_defaults.keys())), - ['bar', 'foo'] - ) + ['bar', 'foo']) diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 811d3a33..7ec9e3c4 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -36,15 +36,16 @@ my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') conf_dir = op.join(fixtures, 'conf') + class CodeTestCase(test.CdistTestCase): def setUp(self): self.local_dir = self.mkdtemp() self.local = local.Local( - target_host=self.target_host, - base_path = self.local_dir, - exec_path = cdist.test.cdist_exec_path, + target_host=self.target_host, + base_path=self.local_dir, + exec_path=cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -52,16 +53,19 @@ class CodeTestCase(test.CdistTestCase): remote_exec = self.remote_exec remote_copy = self.remote_copy self.remote = remote.Remote( - target_host=self.target_host, - remote_exec=remote_exec, + target_host=self.target_host, + remote_exec=remote_exec, remote_copy=remote_copy, base_path=self.remote_dir) self.remote.create_files_dirs() self.code = code.Code(self.target_host, self.local, self.remote) - self.cdist_type = core.CdistType(self.local.type_path, '__dump_environment') - self.cdist_object = core.CdistObject(self.cdist_type, self.local.object_path, 'whatever', self.local.object_marker_name) + self.cdist_type = core.CdistType(self.local.type_path, + '__dump_environment') + self.cdist_object = core.CdistObject( + self.cdist_type, self.local.object_path, 'whatever', + self.local.object_marker_name) self.cdist_object.create() def tearDown(self): @@ -73,14 +77,16 @@ class CodeTestCase(test.CdistTestCase): output_dict = {} for line in output_string.split('\n'): if line: - junk,value = line.split(': ') + junk, value = line.split(': ') key = junk.split(' ')[1] output_dict[key] = value self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) - self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) - self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) + self.assertEqual(output_dict['__object'], + self.cdist_object.absolute_path) + self.assertEqual(output_dict['__object_id'], + self.cdist_object.object_id) self.assertEqual(output_dict['__object_name'], self.cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) @@ -89,29 +95,35 @@ class CodeTestCase(test.CdistTestCase): output_dict = {} for line in output_string.split('\n'): if line: - junk,value = line.split(': ') + junk, value = line.split(': ') key = junk.split(' ')[1] output_dict[key] = value self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__global'], self.local.base_path) self.assertEqual(output_dict['__type'], self.cdist_type.absolute_path) - self.assertEqual(output_dict['__object'], self.cdist_object.absolute_path) - self.assertEqual(output_dict['__object_id'], self.cdist_object.object_id) + self.assertEqual(output_dict['__object'], + self.cdist_object.absolute_path) + self.assertEqual(output_dict['__object_id'], + self.cdist_object.object_id) self.assertEqual(output_dict['__object_name'], self.cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) def test_transfer_code_remote(self): - self.cdist_object.code_remote = self.code.run_gencode_remote(self.cdist_object) + self.cdist_object.code_remote = self.code.run_gencode_remote( + self.cdist_object) self.code.transfer_code_remote(self.cdist_object) - destination = os.path.join(self.remote.object_path, self.cdist_object.code_remote_path) + destination = os.path.join(self.remote.object_path, + self.cdist_object.code_remote_path) self.assertTrue(os.path.isfile(destination)) def test_run_code_local(self): - self.cdist_object.code_local = self.code.run_gencode_local(self.cdist_object) + self.cdist_object.code_local = self.code.run_gencode_local( + self.cdist_object) self.code.run_code_local(self.cdist_object) def test_run_code_remote_environment(self): - self.cdist_object.code_remote = self.code.run_gencode_remote(self.cdist_object) + self.cdist_object.code_remote = self.code.run_gencode_remote( + self.cdist_object) self.code.transfer_code_remote(self.cdist_object) self.code.run_code_remote(self.cdist_object) diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index a36567de..3fd415fd 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -44,6 +44,7 @@ expected_object_names = sorted([ '__second/on-the', '__third/moon']) + class ConfigRunTestCase(test.CdistTestCase): def setUp(self): @@ -65,8 +66,11 @@ class ConfigRunTestCase(test.CdistTestCase): self.objects = [] for cdist_object_name in expected_object_names: cdist_type, cdist_object_id = cdist_object_name.split("/", 1) - cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), self.object_base_path, - self.local.object_marker_name, cdist_object_id) + cdist_object = core.CdistObject(core.CdistType(type_base_path, + cdist_type), + self.object_base_path, + self.local.object_marker_name, + cdist_object_id) cdist_object.create() self.objects.append(cdist_object) @@ -81,8 +85,8 @@ class ConfigRunTestCase(test.CdistTestCase): remote_exec=self.remote_exec, base_path=self.remote_dir) - self.local.object_path = self.object_base_path - self.local.type_path = type_base_path + self.local.object_path = self.object_base_path + self.local.type_path = type_base_path self.config = cdist.config.Config(self.local, self.remote) @@ -95,14 +99,14 @@ class ConfigRunTestCase(test.CdistTestCase): shutil.rmtree(self.temp_dir) def test_dependency_resolution(self): - first = self.object_index['__first/man'] - second = self.object_index['__second/on-the'] - third = self.object_index['__third/moon'] + first = self.object_index['__first/man'] + second = self.object_index['__second/on-the'] + third = self.object_index['__third/moon'] first.requirements = [second.name] second.requirements = [third.name] - # First run: + # First run: # solves first and maybe second (depending on the order in the set) self.config.iterate_once() self.assertTrue(third.state == third.STATE_DONE) @@ -110,11 +114,11 @@ class ConfigRunTestCase(test.CdistTestCase): self.config.iterate_once() self.assertTrue(second.state == second.STATE_DONE) - try: self.config.iterate_once() except cdist.Error: - # Allow failing, because the third run may or may not be unecessary already, + # Allow failing, because the third run may or may not be + # unecessary already, # depending on the order of the objects pass self.assertTrue(first.state == first.STATE_DONE) @@ -123,8 +127,8 @@ class ConfigRunTestCase(test.CdistTestCase): """Ensure an exception is thrown for unresolvable depedencies""" # Create to objects depending on each other - no solution possible - first = self.object_index['__first/man'] - second = self.object_index['__second/on-the'] + first = self.object_index['__first/man'] + second = self.object_index['__second/on-the'] first.requirements = [second.name] second.requirements = [first.name] @@ -153,21 +157,22 @@ class ConfigRunTestCase(test.CdistTestCase): with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): self.config.iterate_until_finished() - def test_dryrun(self): """Test if the dryrun option is working like expected""" drylocal = cdist.exec.local.Local( target_host=self.target_host, base_path=self.local_dir, - #exec_path can not derivated from sys.argv in case of unittest ... - exec_path=os.path.abspath(os.path.join(my_dir,'../../../scripts/cdist')), - initial_manifest=os.path.join(fixtures, 'manifest/dryrun_manifest'), - add_conf_dirs=[ fixtures ] ) + # exec_path can not derivated from sys.argv in case of unittest + exec_path=os.path.abspath(os.path.join( + my_dir, '../../../scripts/cdist')), + initial_manifest=os.path.join(fixtures, + 'manifest/dryrun_manifest'), + add_conf_dirs=[fixtures]) dryrun = cdist.config.Config(drylocal, self.remote, dry_run=True) dryrun.run() # if we are here, dryrun works like expected - + # Currently the resolving code will simply detect that this object does # not exist. It should probably check if the type is a singleton as well diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index ee6c3cea..69ca0fd9 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -40,6 +40,7 @@ my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') conf_dir = op.join(fixtures, 'conf') + class EmulatorTestCase(test.CdistTestCase): def setUp(self): @@ -68,7 +69,8 @@ class EmulatorTestCase(test.CdistTestCase): def test_nonexistent_type_exec(self): argv = ['__does-not-exist'] - self.assertRaises(core.cdist_type.NoSuchTypeError, emulator.Emulator, argv, env=self.env) + self.assertRaises(core.cdist_type.NoSuchTypeError, emulator.Emulator, + argv, env=self.env) def test_nonexistent_type_requirement(self): argv = ['__file', '/tmp/foobar'] @@ -78,7 +80,8 @@ class EmulatorTestCase(test.CdistTestCase): def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] - self.env['require'] = "__file/bad/id/with/%s/inside" % self.local.object_marker_name + self.env['require'] = ( + "__file/bad/id/with/%s/inside") % self.local.object_marker_name emu = emulator.Emulator(argv, env=self.env) self.assertRaises(core.IllegalObjectIdError, emu.run) @@ -123,16 +126,21 @@ class EmulatorTestCase(test.CdistTestCase): emu.run() # now load the objects and verify the require parameter of the objects cdist_type = core.CdistType(self.local.type_path, '__planet') - erde_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'erde') - mars_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'mars') + erde_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'erde') + mars_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'mars') cdist_type = core.CdistType(self.local.type_path, '__file') - file_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, '/tmp/cdisttest') + file_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + '/tmp/cdisttest') # now test the recorded requirements self.assertTrue(len(erde_object.requirements) == 0) self.assertEqual(list(mars_object.requirements), ['__planet/erde']) self.assertEqual(list(file_object.requirements), ['__planet/mars']) # if we get here all is fine + class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): def setUp(self): @@ -170,7 +178,8 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, '__file') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'eggs') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'eggs') reqs = set(('__directory/spam',)) self.assertEqual(reqs, set(cdist_object.requirements)) @@ -189,7 +198,8 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, '__file') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'eggs') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'eggs') reqs = set(('__directory/spam',)) self.assertEqual(reqs, set(cdist_object.requirements)) @@ -214,7 +224,8 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, '__file') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'eggs') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, 'eggs') reqs = set(('__directory/spam', '__directory/spameggs',)) self.assertEqual(reqs, set(cdist_object.requirements)) @@ -240,11 +251,13 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): initial_manifest = os.path.join(self.local.manifest_path, "init") self.manifest.run_initial_manifest(initial_manifest) cdist_type = core.CdistType(self.local.type_path, '__saturn') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, '') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, '') self.manifest.run_type_manifest(cdist_object) expected = ['__planet/Saturn', '__moon/Prometheus'] self.assertEqual(sorted(cdist_object.autorequire), sorted(expected)) + class OverrideTestCase(test.CdistTestCase): def setUp(self): @@ -271,7 +284,7 @@ class OverrideTestCase(test.CdistTestCase): argv = ['__file', '/tmp/foobar'] emu = emulator.Emulator(argv, env=self.env) emu.run() - argv = ['__file', '/tmp/foobar','--mode','404'] + argv = ['__file', '/tmp/foobar', '--mode', '404'] emu = emulator.Emulator(argv, env=self.env) self.assertRaises(cdist.Error, emu.run) @@ -279,7 +292,7 @@ class OverrideTestCase(test.CdistTestCase): argv = ['__file', '/tmp/foobar'] emu = emulator.Emulator(argv, env=self.env) emu.run() - argv = ['__file', '/tmp/foobar','--mode','404'] + argv = ['__file', '/tmp/foobar', '--mode', '404'] self.env['CDIST_OVERRIDE'] = 'on' emu = emulator.Emulator(argv, env=self.env) emu.run() @@ -308,13 +321,17 @@ class ArgumentsTestCase(test.CdistTestCase): shutil.rmtree(self.temp_dir) def test_arguments_with_dashes(self): - argv = ['__arguments_with_dashes', 'some-id', '--with-dash', 'some value'] + argv = ['__arguments_with_dashes', 'some-id', '--with-dash', + 'some value'] os.environ.update(self.env) emu = emulator.Emulator(argv) emu.run() - cdist_type = core.CdistType(self.local.type_path, '__arguments_with_dashes') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'some-id') + cdist_type = core.CdistType(self.local.type_path, + '__arguments_with_dashes') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + 'some-id') self.assertTrue('with-dash' in cdist_object.parameters) def test_boolean(self): @@ -326,7 +343,9 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) self.assertTrue('boolean1' in cdist_object.parameters) self.assertFalse('boolean2' in cdist_object.parameters) # empty file -> True @@ -338,13 +357,16 @@ class ArgumentsTestCase(test.CdistTestCase): type_name = '__arguments_required' object_id = 'some-id' value = 'some value' - argv = [type_name, object_id, '--required1', value, '--required2', value] + argv = [type_name, object_id, '--required1', value, + '--required2', value] os.environ.update(self.env) emu = emulator.Emulator(argv) emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) self.assertTrue('required1' in cdist_object.parameters) self.assertTrue('required2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['required1'], value) @@ -370,7 +392,9 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) self.assertTrue('optional1' in cdist_object.parameters) self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) @@ -385,7 +409,9 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) self.assertTrue('optional1' in cdist_object.parameters) self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) @@ -437,7 +463,9 @@ class StdinTestCase(test.CdistTestCase): ###################################################################### # Create path where stdin should reside at cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) stdin_out_path = os.path.join(cdist_object.absolute_path, 'stdin') ###################################################################### diff --git a/cdist/test/exec/local.py b/cdist/test/exec/local.py index 12a0d47b..2cd8b6db 100644 --- a/cdist/test/exec/local.py +++ b/cdist/test/exec/local.py @@ -39,6 +39,7 @@ conf_dir = op.join(fixtures, "conf") bin_true = "true" bin_false = "false" + class LocalTestCase(test.CdistTestCase): def setUp(self): @@ -59,43 +60,52 @@ class LocalTestCase(test.CdistTestCase): def tearDown(self): shutil.rmtree(self.temp_dir) - ### test api + # test api def test_cache_path(self): - self.assertEqual(self.local.cache_path, os.path.join(self.home_dir, "cache")) + self.assertEqual(self.local.cache_path, + os.path.join(self.home_dir, "cache")) def test_conf_path(self): - self.assertEqual(self.local.conf_path, os.path.join(self.out_path, "conf")) + self.assertEqual(self.local.conf_path, + os.path.join(self.out_path, "conf")) def test_out_path(self): self.assertEqual(self.local.base_path, self.out_path) def test_bin_path(self): - self.assertEqual(self.local.bin_path, os.path.join(self.out_path, "bin")) + self.assertEqual(self.local.bin_path, + os.path.join(self.out_path, "bin")) def test_global_explorer_out_path(self): - self.assertEqual(self.local.global_explorer_out_path, os.path.join(self.out_path, "explorer")) + self.assertEqual(self.local.global_explorer_out_path, + os.path.join(self.out_path, "explorer")) def test_object_path(self): - self.assertEqual(self.local.object_path, os.path.join(self.out_path, "object")) + self.assertEqual(self.local.object_path, + os.path.join(self.out_path, "object")) - ### /test api + # /test api - ### test internal implementation + # test internal implementation def test_global_explorer_path(self): - self.assertEqual(self.local.global_explorer_path, os.path.join(self.out_path, "conf", "explorer")) + self.assertEqual(self.local.global_explorer_path, + os.path.join(self.out_path, "conf", "explorer")) def test_manifest_path(self): - self.assertEqual(self.local.manifest_path, os.path.join(self.out_path, "conf", "manifest")) + self.assertEqual(self.local.manifest_path, + os.path.join(self.out_path, "conf", "manifest")) def test_type_path(self): - self.assertEqual(self.local.type_path, os.path.join(self.out_path, "conf", "type")) + self.assertEqual(self.local.type_path, + os.path.join(self.out_path, "conf", "type")) def test_dist_conf_dir_linking(self): - """Ensure that links are correctly created for types included in distribution""" + """Ensure that links are correctly created for types included + in distribution""" - test_type="__file" + test_type = "__file" link_test_local = local.Local( target_host='localhost', @@ -110,9 +120,10 @@ class LocalTestCase(test.CdistTestCase): self.assertTrue(os.path.isdir(our_type_dir)) def test_added_conf_dir_linking(self): - """Ensure that links are correctly created for types in added conf directories""" + """Ensure that links are correctly created for types in added conf + directories""" - test_type="__cdist_test_type" + test_type = "__cdist_test_type" link_test_local = local.Local( target_host='localhost', @@ -128,9 +139,10 @@ class LocalTestCase(test.CdistTestCase): self.assertTrue(os.path.isdir(our_type_dir)) def test_conf_dir_from_path_linking(self): - """Ensure that links are correctly created for types in conf directories which are defined in CDIST_PATH""" + """Ensure that links are correctly created for types in conf + directories which are defined in CDIST_PATH""" - test_type="__cdist_test_type" + test_type = "__cdist_test_type" os.environ['CDIST_PATH'] = conf_dir @@ -146,7 +158,7 @@ class LocalTestCase(test.CdistTestCase): self.assertTrue(os.path.isdir(our_type_dir)) - ### other tests + # other tests def test_run_success(self): self.local.run([bin_true]) @@ -170,7 +182,8 @@ class LocalTestCase(test.CdistTestCase): handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "echo foobar"]) - self.assertEqual(self.local.run_script(script, return_output=True), "foobar\n") + self.assertEqual(self.local.run_script(script, return_output=True), + "foobar\n") def test_mkdir(self): temp_dir = self.mkdtemp(dir=self.temp_dir) diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index 89e9dbc8..318e4c7b 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -39,26 +39,32 @@ class RemoteTestCase(test.CdistTestCase): user = getpass.getuser() remote_exec = "ssh -o User=%s -q" % user remote_copy = "scp -o User=%s -q" % user - self.remote = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy) + self.remote = remote.Remote(self.target_host, base_path=self.base_path, + remote_exec=remote_exec, + remote_copy=remote_copy) def tearDown(self): shutil.rmtree(self.temp_dir) - ### test api + # test api def test_conf_path(self): - self.assertEqual(self.remote.conf_path, os.path.join(self.base_path, "conf")) + self.assertEqual(self.remote.conf_path, + os.path.join(self.base_path, "conf")) def test_object_path(self): - self.assertEqual(self.remote.object_path, os.path.join(self.base_path, "object")) + self.assertEqual(self.remote.object_path, + os.path.join(self.base_path, "object")) def test_type_path(self): - self.assertEqual(self.remote.type_path, os.path.join(self.base_path, "conf", "type")) + self.assertEqual(self.remote.type_path, + os.path.join(self.base_path, "conf", "type")) def test_global_explorer_path(self): - self.assertEqual(self.remote.global_explorer_path, os.path.join(self.base_path, "conf", "explorer")) + self.assertEqual(self.remote.global_explorer_path, + os.path.join(self.base_path, "conf", "explorer")) - ### /test api + # /test api def test_run_success(self): self.remote.run(['/bin/true']) @@ -76,13 +82,15 @@ class RemoteTestCase(test.CdistTestCase): handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "/bin/false"]) - self.assertRaises(remote.RemoteScriptError, self.remote.run_script, script) + self.assertRaises(remote.RemoteScriptError, self.remote.run_script, + script) def test_run_script_get_output(self): handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "echo foobar"]) - self.assertEqual(self.remote.run_script(script, return_output=True), "foobar\n") + self.assertEqual(self.remote.run_script(script, return_output=True), + "foobar\n") def test_mkdir(self): temp_dir = self.mkdtemp(dir=self.temp_dir) @@ -125,8 +133,10 @@ class RemoteTestCase(test.CdistTestCase): os.chmod(remote_exec_path, 0o755) remote_exec = remote_exec_path remote_copy = "echo" - r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy) - self.assertEqual(r.run('/bin/true', return_output=True), "%s\n" % self.target_host) + r = remote.Remote(self.target_host, base_path=self.base_path, + remote_exec=remote_exec, remote_copy=remote_copy) + self.assertEqual(r.run('/bin/true', return_output=True), + "%s\n" % self.target_host) def test_run_script_target_host_in_env(self): handle, remote_exec_path = self.mkstemp(dir=self.temp_dir) @@ -135,16 +145,21 @@ class RemoteTestCase(test.CdistTestCase): os.chmod(remote_exec_path, 0o755) remote_exec = remote_exec_path remote_copy = "echo" - r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy) + r = remote.Remote(self.target_host, base_path=self.base_path, + remote_exec=remote_exec, remote_copy=remote_copy) handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "/bin/true"]) - self.assertEqual(r.run_script(script, return_output=True), "%s\n" % self.target_host) + self.assertEqual(r.run_script(script, return_output=True), + "%s\n" % self.target_host) def test_run_script_with_env_target_host_in_env(self): handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: - fd.writelines(["#!/bin/sh\n", 'if [ "$__object" ]; then echo $__object; else echo no_env; fi\n']) + fd.writelines([ + "#!/bin/sh\n", + ('if [ "$__object" ]; then echo $__object; ' + 'else echo no_env; fi\n')]) os.chmod(script, 0o755) handle, remote_exec_path = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, 'w') as fd: @@ -152,7 +167,8 @@ class RemoteTestCase(test.CdistTestCase): os.chmod(remote_exec_path, 0o755) remote_exec = remote_exec_path remote_copy = "echo" - r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy) + r = remote.Remote(self.target_host, base_path=self.base_path, + remote_exec=remote_exec, remote_copy=remote_copy) output = r.run_script(script, return_output=True) self.assertEqual(output, "no_env\n") @@ -164,7 +180,8 @@ class RemoteTestCase(test.CdistTestCase): env = { '__object': 'test_object', } - r = remote.Remote(self.target_host, base_path=self.base_path, remote_exec=remote_exec, remote_copy=remote_copy) + r = remote.Remote(self.target_host, base_path=self.base_path, + remote_exec=remote_exec, remote_copy=remote_copy) output = r.run_script(script, env=env, return_output=True) self.assertEqual(output, "test_object\n") diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py index 335d0e32..2ca50b7c 100644 --- a/cdist/test/explorer/__init__.py +++ b/cdist/test/explorer/__init__.py @@ -36,12 +36,13 @@ my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') conf_dir = op.join(fixtures, "conf") + class ExplorerClassTestCase(test.CdistTestCase): def setUp(self): - self.temp_dir = self.mkdtemp() - self.local_path = os.path.join(self.temp_dir, "local") - self.remote_base_path = os.path.join(self.temp_dir, "remote") + self.temp_dir = self.mkdtemp() + self.local_path = os.path.join(self.temp_dir, "local") + self.remote_base_path = os.path.join(self.temp_dir, "remote") os.makedirs(self.remote_base_path) self.local = local.Local( @@ -54,15 +55,15 @@ class ExplorerClassTestCase(test.CdistTestCase): self.local.create_files_dirs() self.remote = remote.Remote( - target_host=self.target_host, + target_host=self.target_host, remote_exec=self.remote_exec, remote_copy=self.remote_copy, base_path=self.remote_base_path) self.remote.create_files_dirs() self.explorer = explorer.Explorer( - self.target_host, - self.local, + self.target_host, + self.local, self.remote) def tearDown(self): @@ -79,7 +80,8 @@ class ExplorerClassTestCase(test.CdistTestCase): self.explorer.transfer_global_explorers() source = self.local.global_explorer_path destination = self.remote.global_explorer_path - self.assertEqual(sorted(os.listdir(source)), sorted(os.listdir(destination))) + self.assertEqual(sorted(os.listdir(source)), + sorted(os.listdir(destination))) def test_run_global_explorer(self): """Checkt that running ONE global explorer works""" @@ -101,14 +103,16 @@ class ExplorerClassTestCase(test.CdistTestCase): def test_list_type_explorer_names(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') expected = cdist_type.explorers - self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), expected) + self.assertEqual(self.explorer.list_type_explorer_names(cdist_type), + expected) def test_transfer_type_explorers(self): """Test if transferring type explorers works""" cdist_type = core.CdistType(self.local.type_path, '__test_type') self.explorer.transfer_type_explorers(cdist_type) source = os.path.join(self.local.type_path, cdist_type.explorer_path) - destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) + destination = os.path.join(self.remote.type_path, + cdist_type.explorer_path) self.assertEqual(os.listdir(source), os.listdir(destination)) def test_transfer_type_explorers_only_once(self): @@ -116,7 +120,8 @@ class ExplorerClassTestCase(test.CdistTestCase): # first transfer self.explorer.transfer_type_explorers(cdist_type) source = os.path.join(self.local.type_path, cdist_type.explorer_path) - destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) + destination = os.path.join(self.remote.type_path, + cdist_type.explorer_path) self.assertEqual(os.listdir(source), os.listdir(destination)) # nuke destination folder content, but recreate directory shutil.rmtree(destination) @@ -127,24 +132,36 @@ class ExplorerClassTestCase(test.CdistTestCase): def test_transfer_object_parameters(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + 'whatever') cdist_object.create() - cdist_object.parameters = {'first': 'first value', 'second': 'second value'} + cdist_object.parameters = { + 'first': 'first value', + 'second': 'second value' + } self.explorer.transfer_object_parameters(cdist_object) - source = os.path.join(self.local.object_path, cdist_object.parameter_path) - destination = os.path.join(self.remote.object_path, cdist_object.parameter_path) - self.assertEqual(sorted(os.listdir(source)), sorted(os.listdir(destination))) + source = os.path.join(self.local.object_path, + cdist_object.parameter_path) + destination = os.path.join(self.remote.object_path, + cdist_object.parameter_path) + self.assertEqual(sorted(os.listdir(source)), + sorted(os.listdir(destination))) def test_run_type_explorer(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + 'whatever') self.explorer.transfer_type_explorers(cdist_type) output = self.explorer.run_type_explorer('world', cdist_object) self.assertEqual(output, 'hello\n') def test_run_type_explorers(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + 'whatever') cdist_object.create() self.explorer.run_type_explorers(cdist_object) self.assertEqual(cdist_object.explorers, {'world': 'hello'}) diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index f9eb8b8a..84c69ce1 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -40,6 +40,7 @@ my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') conf_dir = op.join(fixtures, 'conf') + class ManifestTestCase(test.CdistTestCase): def setUp(self): @@ -51,7 +52,7 @@ class ManifestTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, base_path=out_path, - exec_path = cdist.test.cdist_exec_path, + exec_path=cdist.test.cdist_exec_path, add_conf_dirs=[conf_dir]) self.local.create_files_dirs() @@ -63,7 +64,8 @@ class ManifestTestCase(test.CdistTestCase): shutil.rmtree(self.temp_dir) def test_initial_manifest_environment(self): - initial_manifest = os.path.join(self.local.manifest_path, "dump_environment") + initial_manifest = os.path.join(self.local.manifest_path, + "dump_environment") handle, output_file = self.mkstemp(dir=self.temp_dir) os.close(handle) os.environ['__cdist_test_out'] = output_file @@ -74,18 +76,21 @@ class ManifestTestCase(test.CdistTestCase): output_dict = {} for line in output_string.split('\n'): if line: - key,value = line.split(': ') + key, value = line.split(': ') output_dict[key] = value self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__global'], self.local.base_path) - self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) + self.assertEqual(output_dict['__cdist_type_base_path'], + self.local.type_path) self.assertEqual(output_dict['__manifest'], self.local.manifest_path) self.assertEqual(output_dict['__files'], self.local.files_path) def test_type_manifest_environment(self): cdist_type = core.CdistType(self.local.type_path, '__dump_environment') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + 'whatever') handle, output_file = self.mkstemp(dir=self.temp_dir) os.close(handle) os.environ['__cdist_test_out'] = output_file @@ -96,12 +101,13 @@ class ManifestTestCase(test.CdistTestCase): output_dict = {} for line in output_string.split('\n'): if line: - key,value = line.split(': ') + key, value = line.split(': ') output_dict[key] = value self.assertTrue(output_dict['PATH'].startswith(self.local.bin_path)) self.assertEqual(output_dict['__target_host'], self.local.target_host) self.assertEqual(output_dict['__global'], self.local.base_path) - self.assertEqual(output_dict['__cdist_type_base_path'], self.local.type_path) + self.assertEqual(output_dict['__cdist_type_base_path'], + self.local.type_path) self.assertEqual(output_dict['__type'], cdist_type.absolute_path) self.assertEqual(output_dict['__object'], cdist_object.absolute_path) self.assertEqual(output_dict['__object_id'], cdist_object.object_id) diff --git a/cdist/test/message/__init__.py b/cdist/test/message/__init__.py index 653847f1..61cd5d97 100644 --- a/cdist/test/message/__init__.py +++ b/cdist/test/message/__init__.py @@ -25,14 +25,15 @@ import tempfile from cdist import test import cdist.message + class MessageTestCase(test.CdistTestCase): def setUp(self): - self.prefix="cdist-test" + self.prefix = "cdist-test" self.content = "A very short story" self.tempfile = tempfile.mkstemp()[1] self.message = cdist.message.Message(prefix=self.prefix, - messages=self.tempfile) + messages=self.tempfile) def tearDown(self): os.remove(self.tempfile) @@ -48,7 +49,6 @@ class MessageTestCase(test.CdistTestCase): self.assertIn('__messages_in', env) self.assertIn('__messages_out', env) - def test_copy_content(self): """ Ensure content copying is working diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index 98de09f8..e458fd9e 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -136,7 +136,7 @@ class DirectoryDict(collections.MutableMapping): with open(os.path.join(self.path, key), "w") as fd: if (not hasattr(value, 'strip') and (hasattr(value, '__getitem__') or - hasattr(value, '__iter__'))): + hasattr(value, '__iter__'))): # if it looks like a sequence and quacks like a sequence, # it is a sequence for v in value: @@ -175,14 +175,19 @@ class FileBasedProperty(object): """ :param path: string or callable - Abstract super class. Subclass and set the class member attribute_class accordingly. + Abstract super class. Subclass and set the class member + attribute_class accordingly. Usage with a sublcass: class Foo(object): - # note that the actual DirectoryDict is stored as __parameters on the instance - parameters = DirectoryDictProperty(lambda instance: os.path.join(instance.absolute_path, 'parameter')) - # note that the actual DirectoryDict is stored as __other_dict on the instance + # note that the actual DirectoryDict is stored as __parameters + # on the instance + parameters = DirectoryDictProperty( + lambda instance: os.path.join(instance.absolute_path, + 'parameter')) + # note that the actual DirectoryDict is stored as __other_dict + # on the instance other_dict = DirectoryDictProperty('/tmp/other_dict') def __init__(self):