4c2d273f07
Use one way of string formatting: replace old `%` style with new `str.format`. Resolve #855.
438 lines
18 KiB
Python
438 lines
18 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org)
|
|
# 2012-2013 Steven Armstrong (steven-cdist at armstrong.cc)
|
|
# 2014 Daniel Heule (hda at sfs.biz)
|
|
#
|
|
# This file is part of cdist.
|
|
#
|
|
# cdist is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# cdist is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
#
|
|
|
|
import argparse
|
|
import logging
|
|
import os
|
|
import sys
|
|
import re
|
|
|
|
import cdist
|
|
from cdist import core
|
|
from cdist import flock
|
|
from cdist.core.manifest import Manifest
|
|
|
|
|
|
class MissingRequiredEnvironmentVariableError(cdist.Error):
|
|
def __init__(self, name):
|
|
self.name = name
|
|
self.message = ("Emulator requires the environment variable {} to be "
|
|
"setup").format(self.name)
|
|
|
|
def __str__(self):
|
|
return self.message
|
|
|
|
|
|
class DefaultList(list):
|
|
"""Helper class to allow default values for optional_multiple parameters.
|
|
|
|
@see https://groups.google.com/forum/#!msg/comp.lang.python/sAUvkJEDpRc/RnRymrzJVDYJ # noqa
|
|
"""
|
|
def __copy__(self):
|
|
return []
|
|
|
|
@classmethod
|
|
def create(cls, initial=None):
|
|
if initial:
|
|
return cls(initial.split('\n'))
|
|
|
|
|
|
class Emulator:
|
|
def __init__(self, argv, stdin=sys.stdin.buffer, env=os.environ):
|
|
self.argv = argv
|
|
self.stdin = stdin
|
|
self.env = env
|
|
|
|
self.object_id = ''
|
|
|
|
try:
|
|
self.global_path = self.env['__global']
|
|
self.target_host = (
|
|
self.env['__target_host'],
|
|
self.env['__target_hostname'],
|
|
self.env['__target_fqdn']
|
|
)
|
|
|
|
# Internal variables
|
|
self.object_source = self.env['__cdist_manifest']
|
|
self.type_base_path = self.env['__cdist_type_base_path']
|
|
self.object_marker = self.env['__cdist_object_marker']
|
|
|
|
except KeyError as e:
|
|
raise MissingRequiredEnvironmentVariableError(e.args[0])
|
|
|
|
self.object_base_path = os.path.join(self.global_path, "object")
|
|
self.typeorder_path = os.path.join(self.global_path, "typeorder")
|
|
|
|
self.typeorder_dep_path = os.path.join(self.global_path,
|
|
Manifest.TYPEORDER_DEP_NAME)
|
|
self.order_dep_state_path = os.path.join(self.global_path,
|
|
Manifest.ORDER_DEP_STATE_NAME)
|
|
|
|
self.type_name = os.path.basename(argv[0])
|
|
self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
|
|
|
|
self.__init_log()
|
|
|
|
def run(self):
|
|
"""Emulate type commands (i.e. __file and co)"""
|
|
|
|
self.commandline()
|
|
self.init_object()
|
|
|
|
# locking for parallel execution
|
|
with flock.Flock(self.flock_path):
|
|
self.setup_object()
|
|
self.save_stdin()
|
|
self.record_requirements()
|
|
self.record_auto_requirements()
|
|
self.record_parent_child_relationships()
|
|
self.log.trace("Finished %s %s", self.cdist_object.path,
|
|
self.parameters)
|
|
|
|
def __init_log(self):
|
|
"""Setup logging facility"""
|
|
|
|
if '__cdist_log_level' in self.env:
|
|
try:
|
|
loglevel = self.env['__cdist_log_level']
|
|
level = int(loglevel)
|
|
except ValueError:
|
|
level = logging.WARNING
|
|
else:
|
|
level = logging.WARNING
|
|
self.log = logging.getLogger(self.target_host[0])
|
|
try:
|
|
logging.root.setLevel(level)
|
|
self.log.setLevel(level)
|
|
except (ValueError, TypeError):
|
|
# if invalid __cdist_log_level value
|
|
logging.root.setLevel(logging.WARNING)
|
|
self.log.setLevel(logging.WARNING)
|
|
|
|
colored_log = self.env.get('__cdist_colored_log', 'false')
|
|
cdist.log.CdistFormatter.USE_COLORS = colored_log == 'true'
|
|
|
|
def commandline(self):
|
|
"""Parse command line"""
|
|
|
|
parser = argparse.ArgumentParser(add_help=False,
|
|
argument_default=argparse.SUPPRESS)
|
|
|
|
for parameter in self.cdist_type.required_parameters:
|
|
argument = "--" + parameter
|
|
parser.add_argument(argument, dest=parameter, action='store',
|
|
required=True)
|
|
for parameter in self.cdist_type.required_multiple_parameters:
|
|
argument = "--" + parameter
|
|
parser.add_argument(argument, dest=parameter, action='append',
|
|
required=True)
|
|
for parameter in self.cdist_type.optional_parameters:
|
|
argument = "--" + parameter
|
|
default = self.cdist_type.parameter_defaults.get(parameter, None)
|
|
parser.add_argument(argument, dest=parameter, action='store',
|
|
required=False, default=default)
|
|
for parameter in self.cdist_type.optional_multiple_parameters:
|
|
argument = "--" + parameter
|
|
default = DefaultList.create(
|
|
self.cdist_type.parameter_defaults.get(
|
|
parameter, None))
|
|
parser.add_argument(argument, dest=parameter, action='append',
|
|
required=False, default=default)
|
|
for parameter in self.cdist_type.boolean_parameters:
|
|
argument = "--" + parameter
|
|
parser.add_argument(argument, dest=parameter,
|
|
action='store_const', const='')
|
|
|
|
# If not singleton support one positional parameter
|
|
if not self.cdist_type.is_singleton:
|
|
parser.add_argument("object_id", nargs=1)
|
|
|
|
# And finally parse/verify parameter
|
|
self.args = parser.parse_args(self.argv[1:])
|
|
self.log.trace('Args: %s', self.args)
|
|
|
|
def init_object(self):
|
|
# Initialize object - and ensure it is not in args
|
|
if self.cdist_type.is_singleton:
|
|
self.object_id = ''
|
|
else:
|
|
self.object_id = self.args.object_id[0]
|
|
del self.args.object_id
|
|
|
|
# Instantiate the cdist object we are defining
|
|
self.cdist_object = core.CdistObject(
|
|
self.cdist_type, self.object_base_path, self.object_marker,
|
|
self.object_id)
|
|
lockfname = ('.' + self.cdist_type.name +
|
|
self.object_id + '_' +
|
|
self.object_marker + '.lock')
|
|
lockfname = lockfname.replace(os.sep, '_')
|
|
self.flock_path = os.path.join(self.object_base_path, lockfname)
|
|
|
|
def _object_params_in_context(self):
|
|
''' Get cdist_object parameters dict adopted by context.
|
|
Context consists of cdist_type boolean, optional, required,
|
|
optional_multiple and required_multiple parameters. If parameter
|
|
is multiple parameter then its value is a list.
|
|
This adaptation works on cdist_object.parameters which are read from
|
|
directory based dict where it is unknown what kind of data is in
|
|
file. If there is only one line in the file it is unknown if this
|
|
is a value of required/optional parameter or if it is one value of
|
|
multiple values parameter.
|
|
'''
|
|
params = {}
|
|
if self.cdist_object.exists:
|
|
for param in self.cdist_object.parameters:
|
|
value = ('' if param in self.cdist_type.boolean_parameters
|
|
else self.cdist_object.parameters[param])
|
|
if ((param in self.cdist_type.required_multiple_parameters or
|
|
param in self.cdist_type.optional_multiple_parameters) and
|
|
not isinstance(value, list)):
|
|
value = [value]
|
|
params[param] = value
|
|
return params
|
|
|
|
def setup_object(self):
|
|
# CDIST_ORDER_DEPENDENCY state
|
|
order_dep_on = self._order_dep_on()
|
|
order_dep_defined = "CDIST_ORDER_DEPENDENCY" in self.env
|
|
if not order_dep_defined and order_dep_on:
|
|
self._set_order_dep_state_off()
|
|
if order_dep_defined and not order_dep_on:
|
|
self._set_order_dep_state_on()
|
|
|
|
# Create object with given parameters
|
|
self.parameters = {}
|
|
for key, value in vars(self.args).items():
|
|
if value is not None:
|
|
self.parameters[key] = value
|
|
|
|
if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env:
|
|
obj_params = self._object_params_in_context()
|
|
if obj_params != self.parameters:
|
|
errmsg = ("Object {} already exists with conflicting "
|
|
"parameters:\n{}: {}\n{}: {}").format(
|
|
self.cdist_object.name,
|
|
" ".join(self.cdist_object.source),
|
|
obj_params,
|
|
self.object_source,
|
|
self.parameters)
|
|
raise cdist.Error(errmsg)
|
|
else:
|
|
if self.cdist_object.exists:
|
|
self.log.debug('Object %s override forced with CDIST_OVERRIDE',
|
|
self.cdist_object.name)
|
|
self.cdist_object.create(True)
|
|
else:
|
|
self.cdist_object.create()
|
|
self.cdist_object.parameters = self.parameters
|
|
# Do the following recording even if object exists, but with
|
|
# different requirements.
|
|
|
|
# record the created object in typeorder file
|
|
with open(self.typeorder_path, 'a') as typeorderfile:
|
|
print(self.cdist_object.name, file=typeorderfile)
|
|
# record the created object in parent object typeorder file
|
|
__object_name = self.env.get('__object_name', None)
|
|
depname = self.cdist_object.name
|
|
if __object_name:
|
|
parent = self.cdist_object.object_from_name(__object_name)
|
|
parent.typeorder.append(self.cdist_object.name)
|
|
if self._order_dep_on():
|
|
self.log.trace('[ORDER_DEP] Adding %s to typeorder dep for %s',
|
|
depname, parent.name)
|
|
parent.typeorder_dep.append(depname)
|
|
elif self._order_dep_on():
|
|
self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
|
|
depname)
|
|
self._add_typeorder_dep(depname)
|
|
|
|
# Record / Append source
|
|
self.cdist_object.source.append(self.object_source)
|
|
|
|
chunk_size = 65536
|
|
|
|
def _read_stdin(self):
|
|
return self.stdin.read(self.chunk_size)
|
|
|
|
def save_stdin(self):
|
|
"""If something is written to stdin, save it in the object as
|
|
$__object/stdin so it can be accessed in manifest and gencode-*
|
|
scripts.
|
|
"""
|
|
if not self.stdin.isatty():
|
|
try:
|
|
# go directly to file instead of using CdistObject's api
|
|
# as that does not support streaming
|
|
path = os.path.join(self.cdist_object.absolute_path, 'stdin')
|
|
with open(path, 'wb') as fd:
|
|
chunk = self._read_stdin()
|
|
while chunk:
|
|
fd.write(chunk)
|
|
chunk = self._read_stdin()
|
|
except EnvironmentError as e:
|
|
raise cdist.Error('Failed to read from stdin: {}'.format(e))
|
|
|
|
def record_requirement(self, requirement):
|
|
"""record requirement and return recorded requirement"""
|
|
|
|
# Raises an error, if object cannot be created
|
|
try:
|
|
cdist_object = self.cdist_object.object_from_name(requirement)
|
|
except core.cdist_type.InvalidTypeError as e:
|
|
self.log.error("%s requires object %s, but type %s does not"
|
|
" exist. Defined at %s", self.cdist_object.name,
|
|
requirement, e.name, self.object_source)
|
|
raise
|
|
except core.cdist_object.MissingObjectIdError:
|
|
self.log.error("%s requires object %s without object id."
|
|
" Defined at %s", self.cdist_object.name,
|
|
requirement, self.object_source)
|
|
raise
|
|
|
|
self.log.debug("Recording requirement %s for %s",
|
|
requirement, self.cdist_object.name)
|
|
|
|
# Save the sanitised version, not the user supplied one
|
|
# (__file//bar => __file/bar)
|
|
# This ensures pattern matching is done against sanitised list
|
|
self.cdist_object.requirements.append(cdist_object.name)
|
|
|
|
def _order_dep_on(self):
|
|
return os.path.exists(self.order_dep_state_path)
|
|
|
|
def _set_order_dep_state_on(self):
|
|
self.log.trace('[ORDER_DEP] Setting order dep state on')
|
|
with open(self.order_dep_state_path, 'w'):
|
|
pass
|
|
|
|
def _set_order_dep_state_off(self):
|
|
self.log.trace('[ORDER_DEP] Setting order dep state off')
|
|
# remove order dep state file
|
|
try:
|
|
os.remove(self.order_dep_state_path)
|
|
except FileNotFoundError:
|
|
pass
|
|
# remove typeorder dep file
|
|
try:
|
|
os.remove(self.typeorder_dep_path)
|
|
except FileNotFoundError:
|
|
pass
|
|
|
|
def _add_typeorder_dep(self, name):
|
|
with open(self.typeorder_dep_path, 'a') as f:
|
|
print(name, file=f)
|
|
|
|
def _read_typeorder_dep(self):
|
|
try:
|
|
with open(self.typeorder_dep_path, 'r') as f:
|
|
return f.readlines()
|
|
except FileNotFoundError:
|
|
return []
|
|
|
|
def record_requirements(self):
|
|
"""Record requirements."""
|
|
|
|
order_dep_on = self._order_dep_on()
|
|
|
|
# Inject the predecessor, but not if its an override
|
|
# (this would leed to an circular dependency)
|
|
if (order_dep_on and 'CDIST_OVERRIDE' not in self.env):
|
|
try:
|
|
# __object_name is the name of the object whose type
|
|
# manifest is currently executed
|
|
__object_name = self.env.get('__object_name', None)
|
|
# load object name created befor this one from typeorder
|
|
# dep file
|
|
if __object_name:
|
|
parent = self.cdist_object.object_from_name(
|
|
__object_name)
|
|
typeorder = parent.typeorder_dep
|
|
else:
|
|
typeorder = self._read_typeorder_dep()
|
|
# get the type created before this one
|
|
lastcreatedtype = typeorder[-2].strip()
|
|
if 'require' in self.env:
|
|
if lastcreatedtype not in self.env['require']:
|
|
self.env['require'] += " " + lastcreatedtype
|
|
else:
|
|
self.env['require'] = lastcreatedtype
|
|
self.log.debug("Injecting require for"
|
|
" CDIST_ORDER_DEPENDENCY: %s for %s",
|
|
lastcreatedtype, self.cdist_object.name)
|
|
except IndexError:
|
|
# if no second last line, we are on the first type,
|
|
# so do not set a requirement
|
|
pass
|
|
|
|
if "require" in self.env:
|
|
requirements = self.env['require']
|
|
self.log.debug("reqs = %s", requirements)
|
|
for requirement in self._parse_require(requirements):
|
|
# Ignore empty fields - probably the only field anyway
|
|
if len(requirement) == 0:
|
|
continue
|
|
self.record_requirement(requirement)
|
|
|
|
def _parse_require(self, require):
|
|
return re.split(r'[ \t\n]+', require)
|
|
|
|
def record_auto_requirements(self):
|
|
"""An object shall automatically depend on all objects that it
|
|
defined in it's type manifest.
|
|
"""
|
|
# __object_name is the name of the object whose type manifest is
|
|
# currently executed
|
|
__object_name = self.env.get('__object_name', None)
|
|
if __object_name:
|
|
# The object whose type manifest is currently run
|
|
parent = self.cdist_object.object_from_name(__object_name)
|
|
# The object currently being defined
|
|
current_object = self.cdist_object
|
|
# As parent defined current_object it shall automatically
|
|
# depend on it.
|
|
# But only if the user hasn't said otherwise.
|
|
# Must prevent circular dependencies.
|
|
if parent.name not in current_object.requirements:
|
|
self.log.debug("Recording autorequirement %s for %s",
|
|
current_object.name, parent.name)
|
|
parent.autorequire.append(current_object.name)
|
|
|
|
def record_parent_child_relationships(self):
|
|
# __object_name is the name of the object whose type manifest is
|
|
# currently executed
|
|
__object_name = self.env.get('__object_name', None)
|
|
if __object_name:
|
|
# The object whose type manifest is currently run
|
|
parent = self.cdist_object.object_from_name(__object_name)
|
|
# The object currently being defined
|
|
current_object = self.cdist_object
|
|
if current_object.name not in parent.children:
|
|
self.log.debug("Recording child %s for %s",
|
|
current_object.name, parent.name)
|
|
parent.children.append(current_object.name)
|
|
if parent.name not in current_object.parents:
|
|
self.log.debug("Recording parent %s for %s",
|
|
parent.name, current_object.name)
|
|
current_object.parents.append(parent.name)
|