forked from ungleich-public/cdist
da274e5ef3
CDIST_ORDER_DEPENDENCY now defines type order dependency context.
cdist (emulator) maintains global state variables, as files,
order_dep_state and typeorder_dep, and per object state variable,
as file, typeorder_dep.
If order_dep_state exists then this defines that order dependency is
turned on.
If order_dep_state does not exist then order dependency is turned off.
If order dependency is on then objects created after it is turned on are
recorded into:
* global typeorder_dep, in case of init manifest
* object's typeorder_dep, in case of type's manifest.
If order dependency is on then requirement is injected, where object
created before current, is read from:
* global typeorder_dep, in case of init manifest
* object's typeorder_dep, in case of type's manifest.
Every time order dependency is turned off, typeorder_dep files are
removed, which means that type order list is cleared, context is
cleaned.
In the end cdist cleans after itself, i.e. mentioned files are removed.
When running type manifest is finished typeorder_dep file is removed.
When running config finishes global typeorder_dep and order_dep_state
files are removed.
Global type order recording is untouched.
Furthermore, for completeness, type order is now recorded for each object
too.
437 lines
18 KiB
Python
437 lines
18 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
|
|
# 2012-2013 Steven Armstrong (steven-cdist at armstrong.cc)
|
|
# 2014 Daniel Heule (hda at sfs.biz)
|
|
#
|
|
# This file is part of cdist.
|
|
#
|
|
# cdist is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# cdist is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
#
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import sys
|
|
|
|
import cdist
|
|
from cdist import core
|
|
from cdist import flock
|
|
from cdist.core.manifest import Manifest
|
|
|
|
|
|
class MissingRequiredEnvironmentVariableError(cdist.Error):
|
|
def __init__(self, name):
|
|
self.name = name
|
|
self.message = ("Emulator requires the environment variable %s to be "
|
|
"setup" % self.name)
|
|
|
|
def __str__(self):
|
|
return self.message
|
|
|
|
|
|
class DefaultList(list):
|
|
"""Helper class to allow default values for optional_multiple parameters.
|
|
|
|
@see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ # noqa
|
|
"""
|
|
def __copy__(self):
|
|
return []
|
|
|
|
@classmethod
|
|
def create(cls, initial=None):
|
|
if initial:
|
|
return cls(initial.split('\n'))
|
|
|
|
|
|
class Emulator(object):
|
|
def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
|
|
self.argv = argv
|
|
self.stdin = stdin
|
|
self.env = env
|
|
|
|
self.object_id = ''
|
|
|
|
try:
|
|
self.global_path = self.env['__global']
|
|
self.target_host = (
|
|
self.env['__target_host'],
|
|
self.env['__target_hostname'],
|
|
self.env['__target_fqdn']
|
|
)
|
|
|
|
# Internal variables
|
|
self.object_source = self.env['__cdist_manifest']
|
|
self.type_base_path = self.env['__cdist_type_base_path']
|
|
self.object_marker = self.env['__cdist_object_marker']
|
|
|
|
except KeyError as e:
|
|
raise MissingRequiredEnvironmentVariableError(e.args[0])
|
|
|
|
self.object_base_path = os.path.join(self.global_path, "object")
|
|
self.typeorder_path = os.path.join(self.global_path, "typeorder")
|
|
|
|
self.typeorder_dep_path = os.path.join(self.global_path,
|
|
Manifest.TYPEORDER_DEP_NAME)
|
|
self.order_dep_state_path = os.path.join(self.global_path,
|
|
Manifest.ORDER_DEP_STATE_NAME)
|
|
|
|
self.type_name = os.path.basename(argv[0])
|
|
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
|
|
|
|
# If set then object alreay exists and this var holds existing
|
|
# requirements.
|
|
self._existing_reqs = None
|
|
|
|
self.__init_log()
|
|
|
|
def run(self):
|
|
"""Emulate type commands (i.e. __file and co)"""
|
|
|
|
self.commandline()
|
|
self.init_object()
|
|
|
|
# locking for parallel execution
|
|
with flock.Flock(self.flock_path):
|
|
self.setup_object()
|
|
self.save_stdin()
|
|
self.record_requirements()
|
|
self.record_auto_requirements()
|
|
self.log.trace("Finished %s %s" % (
|
|
self.cdist_object.path, self.parameters))
|
|
|
|
def __init_log(self):
|
|
"""Setup logging facility"""
|
|
|
|
if '__cdist_log_level' in self.env:
|
|
try:
|
|
loglevel = self.env['__cdist_log_level']
|
|
level = int(loglevel)
|
|
except ValueError:
|
|
level = logging.WARNING
|
|
else:
|
|
level = logging.WARNING
|
|
try:
|
|
logging.root.setLevel(level)
|
|
except (ValueError, TypeError):
|
|
# if invalid __cdist_log_level value
|
|
logging.root.setLevel(logging.WARNING)
|
|
|
|
self.log = logging.getLogger(self.target_host[0])
|
|
|
|
def commandline(self):
|
|
"""Parse command line"""
|
|
|
|
parser = argparse.ArgumentParser(add_help=False,
|
|
argument_default=argparse.SUPPRESS)
|
|
|
|
for parameter in self.cdist_type.required_parameters:
|
|
argument = "--" + parameter
|
|
parser.add_argument(argument, dest=parameter, action='store',
|
|
required=True)
|
|
for parameter in self.cdist_type.required_multiple_parameters:
|
|
argument = "--" + parameter
|
|
parser.add_argument(argument, dest=parameter, action='append',
|
|
required=True)
|
|
for parameter in self.cdist_type.optional_parameters:
|
|
argument = "--" + parameter
|
|
default = self.cdist_type.parameter_defaults.get(parameter, None)
|
|
parser.add_argument(argument, dest=parameter, action='store',
|
|
required=False, default=default)
|
|
for parameter in self.cdist_type.optional_multiple_parameters:
|
|
argument = "--" + parameter
|
|
default = DefaultList.create(
|
|
self.cdist_type.parameter_defaults.get(
|
|
parameter, None))
|
|
parser.add_argument(argument, dest=parameter, action='append',
|
|
required=False, default=default)
|
|
for parameter in self.cdist_type.boolean_parameters:
|
|
argument = "--" + parameter
|
|
parser.add_argument(argument, dest=parameter,
|
|
action='store_const', const='')
|
|
|
|
# If not singleton support one positional parameter
|
|
if not self.cdist_type.is_singleton:
|
|
parser.add_argument("object_id", nargs=1)
|
|
|
|
# And finally parse/verify parameter
|
|
self.args = parser.parse_args(self.argv[1:])
|
|
self.log.trace('Args: %s' % self.args)
|
|
|
|
def init_object(self):
|
|
# Initialize object - and ensure it is not in args
|
|
if self.cdist_type.is_singleton:
|
|
self.object_id = ''
|
|
else:
|
|
self.object_id = self.args.object_id[0]
|
|
del self.args.object_id
|
|
|
|
# Instantiate the cdist object we are defining
|
|
self.cdist_object = core.CdistObject(
|
|
self.cdist_type, self.object_base_path, self.object_marker,
|
|
self.object_id)
|
|
lockfname = ('.' + self.cdist_type.name +
|
|
self.object_id + '_' +
|
|
self.object_marker + '.lock')
|
|
lockfname = lockfname.replace(os.sep, '_')
|
|
self.flock_path = os.path.join(self.object_base_path, lockfname)
|
|
|
|
def _object_params_in_context(self):
|
|
''' Get cdist_object parameters dict adopted by context.
|
|
Context consists of cdist_type boolean, optional, required,
|
|
optional_multiple and required_multiple parameters. If parameter
|
|
is multiple parameter then its value is a list.
|
|
This adaptation works on cdist_object.parameters which are read from
|
|
directory based dict where it is unknown what kind of data is in
|
|
file. If there is only one line in the file it is unknown if this
|
|
is a value of required/optional parameter or if it is one value of
|
|
multiple values parameter.
|
|
'''
|
|
params = {}
|
|
if self.cdist_object.exists:
|
|
for param in self.cdist_object.parameters:
|
|
value = ('' if param in self.cdist_type.boolean_parameters
|
|
else self.cdist_object.parameters[param])
|
|
if ((param in self.cdist_type.required_multiple_parameters or
|
|
param in self.cdist_type.optional_multiple_parameters) and
|
|
not isinstance(value, list)):
|
|
value = [value]
|
|
params[param] = value
|
|
return params
|
|
|
|
def setup_object(self):
|
|
# CDIST_ORDER_DEPENDENCY state
|
|
order_dep_on = self._order_dep_on()
|
|
order_dep_defined = "CDIST_ORDER_DEPENDENCY" in self.env
|
|
if not order_dep_defined and order_dep_on:
|
|
self._set_order_dep_state_off()
|
|
if order_dep_defined and not order_dep_on:
|
|
self._set_order_dep_state_on()
|
|
|
|
# Create object with given parameters
|
|
self.parameters = {}
|
|
for key, value in vars(self.args).items():
|
|
if value is not None:
|
|
self.parameters[key] = value
|
|
|
|
if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
|
|
# Make existing requirements a set so that we can compare it
|
|
# later with new requirements.
|
|
self._existing_reqs = set(self.cdist_object.requirements)
|
|
obj_params = self._object_params_in_context()
|
|
if obj_params != self.parameters:
|
|
errmsg = ("Object %s already exists with conflicting "
|
|
"parameters:\n%s: %s\n%s: %s" % (
|
|
self.cdist_object.name,
|
|
" ".join(self.cdist_object.source),
|
|
obj_params,
|
|
self.object_source,
|
|
self.parameters))
|
|
raise cdist.Error(errmsg)
|
|
else:
|
|
if self.cdist_object.exists:
|
|
self.log.debug(('Object %s override forced with '
|
|
'CDIST_OVERRIDE'), self.cdist_object.name)
|
|
self.cdist_object.create(True)
|
|
else:
|
|
self.cdist_object.create()
|
|
self.cdist_object.parameters = self.parameters
|
|
# record the created object in typeorder file
|
|
with open(self.typeorder_path, 'a') as typeorderfile:
|
|
print(self.cdist_object.name, file=typeorderfile)
|
|
# record the created object in parent object typeorder file
|
|
__object_name = self.env.get('__object_name', None)
|
|
depname = self.cdist_object.name
|
|
if __object_name:
|
|
parent = self.cdist_object.object_from_name(__object_name)
|
|
parent.typeorder.append(self.cdist_object.name)
|
|
if self._order_dep_on():
|
|
self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
|
|
' for %s'), depname, parent.name)
|
|
parent.typeorder_dep.append(depname)
|
|
elif self._order_dep_on():
|
|
self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
|
|
depname)
|
|
self._add_typeorder_dep(depname)
|
|
|
|
# Record / Append source
|
|
self.cdist_object.source.append(self.object_source)
|
|
|
|
chunk_size = 65536
|
|
|
|
def _read_stdin(self):
|
|
return self.stdin.read(self.chunk_size)
|
|
|
|
def save_stdin(self):
|
|
"""If something is written to stdin, save it in the object as
|
|
$__object/stdin so it can be accessed in manifest and gencode-*
|
|
scripts.
|
|
"""
|
|
if not self.stdin.isatty():
|
|
try:
|
|
# go directly to file instead of using CdistObject's api
|
|
# as that does not support streaming
|
|
path = os.path.join(self.cdist_object.absolute_path, 'stdin')
|
|
with open(path, 'wb') as fd:
|
|
chunk = self._read_stdin()
|
|
while chunk:
|
|
fd.write(chunk)
|
|
chunk = self._read_stdin()
|
|
except EnvironmentError as e:
|
|
raise cdist.Error('Failed to read from stdin: %s' % e)
|
|
|
|
def record_requirement(self, requirement):
|
|
"""record requirement and return recorded requirement"""
|
|
|
|
# Raises an error, if object cannot be created
|
|
try:
|
|
cdist_object = self.cdist_object.object_from_name(requirement)
|
|
except core.cdist_type.InvalidTypeError as e:
|
|
self.log.error(("%s requires object %s, but type %s does not"
|
|
" exist. Defined at %s" % (
|
|
self.cdist_object.name,
|
|
requirement, e.name, self.object_source)))
|
|
raise
|
|
except core.cdist_object.MissingObjectIdError:
|
|
self.log.error(("%s requires object %s without object id."
|
|
" Defined at %s" % (self.cdist_object.name,
|
|
requirement,
|
|
self.object_source)))
|
|
raise
|
|
|
|
self.log.debug("Recording requirement %s for %s",
|
|
requirement, self.cdist_object.name)
|
|
|
|
# Save the sanitised version, not the user supplied one
|
|
# (__file//bar => __file/bar)
|
|
# This ensures pattern matching is done against sanitised list
|
|
self.cdist_object.requirements.append(cdist_object.name)
|
|
|
|
return cdist_object.name
|
|
|
|
def _order_dep_on(self):
|
|
return os.path.exists(self.order_dep_state_path)
|
|
|
|
def _set_order_dep_state_on(self):
|
|
self.log.trace('[ORDER_DEP] Setting order dep state on')
|
|
with open(self.order_dep_state_path, 'w'):
|
|
pass
|
|
|
|
def _set_order_dep_state_off(self):
|
|
self.log.trace('[ORDER_DEP] Setting order dep state off')
|
|
# remove order dep state file
|
|
try:
|
|
os.remove(self.order_dep_state_path)
|
|
except FileNotFoundError:
|
|
pass
|
|
# remove typeorder dep file
|
|
try:
|
|
os.remove(self.typeorder_dep_path)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
def _add_typeorder_dep(self, name):
|
|
with open(self.typeorder_dep_path, 'a') as f:
|
|
print(name, file=f)
|
|
|
|
def _read_typeorder_dep(self):
|
|
try:
|
|
with open(self.typeorder_dep_path, 'r') as f:
|
|
return f.readlines()
|
|
except FileNotFoundError:
|
|
return []
|
|
|
|
def record_requirements(self):
|
|
"""Record requirements."""
|
|
|
|
order_dep_on = self._order_dep_on()
|
|
|
|
# Inject the predecessor, but not if its an override
|
|
# (this would leed to an circular dependency)
|
|
if (order_dep_on and 'CDIST_OVERRIDE' not in self.env):
|
|
try:
|
|
# __object_name is the name of the object whose type
|
|
# manifest is currently executed
|
|
__object_name = self.env.get('__object_name', None)
|
|
# load object name created befor this one from typeorder
|
|
# dep file
|
|
if __object_name:
|
|
parent = self.cdist_object.object_from_name(
|
|
__object_name)
|
|
typeorder = parent.typeorder_dep
|
|
else:
|
|
typeorder = self._read_typeorder_dep()
|
|
# get the type created before this one
|
|
lastcreatedtype = typeorder[-2].strip()
|
|
if 'require' in self.env:
|
|
if lastcreatedtype not in self.env['require']:
|
|
self.env['require'] += " " + lastcreatedtype
|
|
else:
|
|
self.env['require'] = lastcreatedtype
|
|
self.log.debug(("Injecting require for "
|
|
"CDIST_ORDER_DEPENDENCY: %s for %s"),
|
|
lastcreatedtype,
|
|
self.cdist_object.name)
|
|
except IndexError:
|
|
# if no second last line, we are on the first type,
|
|
# so do not set a requirement
|
|
pass
|
|
|
|
reqs = set()
|
|
if "require" in self.env:
|
|
requirements = self.env['require']
|
|
self.log.debug("reqs = " + requirements)
|
|
for requirement in requirements.split(" "):
|
|
# Ignore empty fields - probably the only field anyway
|
|
if len(requirement) == 0:
|
|
continue
|
|
object_name = self.record_requirement(requirement)
|
|
reqs.add(object_name)
|
|
if self._existing_reqs is not None:
|
|
# If object exists then compare existing and new requirements.
|
|
if self._existing_reqs != reqs:
|
|
warnmsg = ("Object {} already exists with requirements:\n"
|
|
"{}: {}\n"
|
|
"{}: {}\n"
|
|
"Dependency resolver could not handle dependencies "
|
|
"as expected.".format(
|
|
self.cdist_object.name,
|
|
" ".join(self.cdist_object.source),
|
|
self._existing_reqs,
|
|
self.object_source,
|
|
reqs
|
|
))
|
|
self.log.warning(warnmsg)
|
|
|
|
def record_auto_requirements(self):
|
|
"""An object shall automatically depend on all objects that it
|
|
defined in it's type manifest.
|
|
"""
|
|
# __object_name is the name of the object whose type manifest is
|
|
# currently executed
|
|
__object_name = self.env.get('__object_name', None)
|
|
if __object_name:
|
|
# The object whose type manifest is currently run
|
|
parent = self.cdist_object.object_from_name(__object_name)
|
|
# The object currently being defined
|
|
current_object = self.cdist_object
|
|
# As parent defined current_object it shall automatically
|
|
# depend on it.
|
|
# But only if the user hasn't said otherwise.
|
|
# Must prevent circular dependencies.
|
|
if parent.name not in current_object.requirements:
|
|
self.log.debug("Recording autorequirement %s for %s",
|
|
current_object.name, parent.name)
|
|
parent.autorequire.append(current_object.name)
|