Redefine/reimplement CDIST_ORDER_DEPENDENCY

CDIST_ORDER_DEPENDENCY now defines type order dependency context.
cdist (emulator) maintains global state variables, as files,
order_dep_state and typeorder_dep, and per object state variable,
as file, typeorder_dep.

If order_dep_state exists then this defines that order dependency is
turned on.
If order_dep_state does not exist then order dependency is turned off.

If order dependency is on then objects created after it is turned on are
recorded into:
    * global typeorder_dep, in case of init manifest
    * object's typeorder_dep, in case of type's manifest.

If order dependency is on then requirement is injected, where object
created before current, is read from:
    * global typeorder_dep, in case of init manifest
    * object's typeorder_dep, in case of type's manifest.

Every time order dependency is turned off, typeorder_dep files are
removed, which means that type order list is cleared, context is
cleaned.

In the end cdist cleans after itself, i.e. mentioned files are removed.

When running type manifest is finished typeorder_dep file is removed.
When running config finishes global typeorder_dep and order_dep_state
files are removed.

Global type order recording is untouched.
Furthermore, for completeness, type order is now recorded for each object
too.
This commit is contained in:
Darko Poljak 2019-11-23 00:25:51 +01:00
parent f3bd439c43
commit da274e5ef3
5 changed files with 157 additions and 38 deletions

View file

@ -124,6 +124,7 @@ class Config(object):
"""Remove files and directories for the run""" """Remove files and directories for the run"""
if self.remove_remote_files_dirs: if self.remove_remote_files_dirs:
self._remove_remote_files_dirs() self._remove_remote_files_dirs()
self.manifest.cleanup()
@staticmethod @staticmethod
def hosts(source): def hosts(source):
@ -787,6 +788,9 @@ class Config(object):
self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) self.explorer.run_type_explorers(cdist_object, transfer_type_explorers)
try: try:
self.manifest.run_type_manifest(cdist_object) self.manifest.run_type_manifest(cdist_object)
self.log.trace("[ORDER_DEP] Removing order dep files for %s",
cdist_object)
cdist_object.cleanup()
cdist_object.state = core.CdistObject.STATE_PREPARED cdist_object.state = core.CdistObject.STATE_PREPARED
except cdist.Error as e: except cdist.Error as e:
raise cdist.CdistObjectError(cdist_object, e) raise cdist.CdistObjectError(cdist_object, e)

View file

@ -243,6 +243,16 @@ class CdistObject(object):
lambda obj: os.path.join(obj.base_path, obj.code_local_path)) lambda obj: os.path.join(obj.base_path, obj.code_local_path))
code_remote = fsproperty.FileStringProperty( code_remote = fsproperty.FileStringProperty(
lambda obj: os.path.join(obj.base_path, obj.code_remote_path)) lambda obj: os.path.join(obj.base_path, obj.code_remote_path))
typeorder = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, 'typeorder'))
typeorder_dep = fsproperty.FileListProperty(
lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep'))
def cleanup(self):
try:
os.remove(os.path.join(self.absolute_path, 'typeorder_dep'))
except FileNotFoundError:
pass
@property @property
def exists(self): def exists(self):

View file

@ -96,6 +96,10 @@ class Manifest(object):
"""Executes cdist manifests. """Executes cdist manifests.
""" """
ORDER_DEP_STATE_NAME = 'order_dep_state'
TYPEORDER_DEP_NAME = 'typeorder_dep'
def __init__(self, target_host, local, dry_run=False): def __init__(self, target_host, local, dry_run=False):
self.target_host = target_host self.target_host = target_host
self.local = local self.local = local
@ -212,3 +216,13 @@ class Manifest(object):
type_manifest, type_manifest,
env=self.env_type_manifest(cdist_object), env=self.env_type_manifest(cdist_object),
message_prefix=message_prefix) message_prefix=message_prefix)
def cleanup(self):
def _rm_file(fname):
try:
self.log.trace("[ORDER_DEP] Removing %s", fname)
os.remove(os.path.join(self.local.base_path, fname))
except FileNotFoundError:
pass
_rm_file(Manifest.ORDER_DEP_STATE_NAME)
_rm_file(Manifest.TYPEORDER_DEP_NAME)

View file

@ -29,6 +29,7 @@ import sys
import cdist import cdist
from cdist import core from cdist import core
from cdist import flock from cdist import flock
from cdist.core.manifest import Manifest
class MissingRequiredEnvironmentVariableError(cdist.Error): class MissingRequiredEnvironmentVariableError(cdist.Error):
@ -82,6 +83,11 @@ class Emulator(object):
self.object_base_path = os.path.join(self.global_path, "object") self.object_base_path = os.path.join(self.global_path, "object")
self.typeorder_path = os.path.join(self.global_path, "typeorder") self.typeorder_path = os.path.join(self.global_path, "typeorder")
self.typeorder_dep_path = os.path.join(self.global_path,
Manifest.TYPEORDER_DEP_NAME)
self.order_dep_state_path = os.path.join(self.global_path,
Manifest.ORDER_DEP_STATE_NAME)
self.type_name = os.path.basename(argv[0]) self.type_name = os.path.basename(argv[0])
self.cdist_type = core.CdistType(self.type_base_path, self.type_name) self.cdist_type = core.CdistType(self.type_base_path, self.type_name)
@ -206,6 +212,14 @@ class Emulator(object):
return params return params
def setup_object(self): def setup_object(self):
# CDIST_ORDER_DEPENDENCY state
order_dep_on = self._order_dep_on()
order_dep_defined = "CDIST_ORDER_DEPENDENCY" in self.env
if not order_dep_defined and order_dep_on:
self._set_order_dep_state_off()
if order_dep_defined and not order_dep_on:
self._set_order_dep_state_on()
# Create object with given parameters # Create object with given parameters
self.parameters = {} self.parameters = {}
for key, value in vars(self.args).items(): for key, value in vars(self.args).items():
@ -237,6 +251,20 @@ class Emulator(object):
# record the created object in typeorder file # record the created object in typeorder file
with open(self.typeorder_path, 'a') as typeorderfile: with open(self.typeorder_path, 'a') as typeorderfile:
print(self.cdist_object.name, file=typeorderfile) print(self.cdist_object.name, file=typeorderfile)
# record the created object in parent object typeorder file
__object_name = self.env.get('__object_name', None)
depname = self.cdist_object.name
if __object_name:
parent = self.cdist_object.object_from_name(__object_name)
parent.typeorder.append(self.cdist_object.name)
if self._order_dep_on():
self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep'
' for %s'), depname, parent.name)
parent.typeorder_dep.append(depname)
elif self._order_dep_on():
self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep',
depname)
self._add_typeorder_dep(depname)
# Record / Append source # Record / Append source
self.cdist_object.source.append(self.object_source) self.cdist_object.source.append(self.object_source)
@ -293,45 +321,73 @@ class Emulator(object):
return cdist_object.name return cdist_object.name
def _order_dep_on(self):
return os.path.exists(self.order_dep_state_path)
def _set_order_dep_state_on(self):
self.log.trace('[ORDER_DEP] Setting order dep state on')
with open(self.order_dep_state_path, 'w'):
pass
def _set_order_dep_state_off(self):
self.log.trace('[ORDER_DEP] Setting order dep state off')
# remove order dep state file
try:
os.remove(self.order_dep_state_path)
except FileNotFoundError:
pass
# remove typeorder dep file
try:
os.remove(self.typeorder_dep_path)
except FileNotFoundError:
pass
def _add_typeorder_dep(self, name):
with open(self.typeorder_dep_path, 'a') as f:
print(name, file=f)
def _read_typeorder_dep(self):
try:
with open(self.typeorder_dep_path, 'r') as f:
return f.readlines()
except FileNotFoundError:
return []
def record_requirements(self): def record_requirements(self):
"""Record requirements.""" """Record requirements."""
order_dep_on = self._order_dep_on()
# Inject the predecessor, but not if its an override # Inject the predecessor, but not if its an override
# (this would leed to an circular dependency) # (this would leed to an circular dependency)
if ("CDIST_ORDER_DEPENDENCY" in self.env and if (order_dep_on and 'CDIST_OVERRIDE' not in self.env):
'CDIST_OVERRIDE' not in self.env): try:
# load object name created befor this one from typeorder file ... # __object_name is the name of the object whose type
with open(self.typeorder_path, 'r') as typecreationfile: # manifest is currently executed
typecreationorder = typecreationfile.readlines() __object_name = self.env.get('__object_name', None)
# get the type created before this one ... # load object name created befor this one from typeorder
try: # dep file
lastcreatedtype = typecreationorder[-2].strip() if __object_name:
# __object_name is the name of the object whose type parent = self.cdist_object.object_from_name(
# manifest is currently executed __object_name)
__object_name = self.env.get('__object_name', None) typeorder = parent.typeorder_dep
if lastcreatedtype == __object_name: else:
self.log.debug(("Not injecting require for " typeorder = self._read_typeorder_dep()
"CDIST_ORDER_DEPENDENCY: %s for %s," # get the type created before this one
" %s's type manifest is currently" lastcreatedtype = typeorder[-2].strip()
" being executed"), if 'require' in self.env:
lastcreatedtype, if lastcreatedtype not in self.env['require']:
self.cdist_object.name, self.env['require'] += " " + lastcreatedtype
lastcreatedtype) else:
else: self.env['require'] = lastcreatedtype
if 'require' in self.env: self.log.debug(("Injecting require for "
appendix = " " + lastcreatedtype "CDIST_ORDER_DEPENDENCY: %s for %s"),
if appendix not in self.env['require']: lastcreatedtype,
self.env['require'] += appendix self.cdist_object.name)
else: except IndexError:
self.env['require'] = lastcreatedtype # if no second last line, we are on the first type,
self.log.debug(("Injecting require for " # so do not set a requirement
"CDIST_ORDER_DEPENDENCY: %s for %s"), pass
lastcreatedtype,
self.cdist_object.name)
except IndexError:
# if no second last line, we are on the first type,
# so do not set a requirement
pass
reqs = set() reqs = set()
if "require" in self.env: if "require" in self.env:

View file

@ -24,8 +24,6 @@
import io import io
import os import os
import shutil import shutil
import string
import filecmp
import random import random
import logging import logging
@ -34,7 +32,6 @@ from cdist import test
from cdist.exec import local from cdist.exec import local
from cdist import emulator from cdist import emulator
from cdist import core from cdist import core
from cdist import config
import os.path as op import os.path as op
my_dir = op.abspath(op.dirname(__file__)) my_dir = op.abspath(op.dirname(__file__))
@ -115,7 +112,7 @@ class EmulatorTestCase(test.CdistTestCase):
def test_requirement_pattern(self): def test_requirement_pattern(self):
argv = ['__file', '/tmp/foobar'] argv = ['__file', '/tmp/foobar']
self.env['require'] = '__file/etc/*' self.env['require'] = '__file/etc/*'
emu = emulator.Emulator(argv, env=self.env) emulator.Emulator(argv, env=self.env)
# if we get here all is fine # if we get here all is fine
def test_loglevel(self): def test_loglevel(self):
@ -172,6 +169,44 @@ class EmulatorTestCase(test.CdistTestCase):
self.assertEqual(list(file_object.requirements), ['__planet/mars']) self.assertEqual(list(file_object.requirements), ['__planet/mars'])
# if we get here all is fine # if we get here all is fine
def test_order_dependency_context(self):
test_seq = ('A', True, 'B', 'C', 'D', False, 'E', 'F', True, 'G',
'H', False, 'I', )
expected_requirements = {
'C': set(('__planet/B', )),
'D': set(('__planet/C', )),
'H': set(('__planet/G', )),
}
# Ensure env var is not in env
if 'CDIST_ORDER_DEPENDENCY' in self.env:
del self.env['CDIST_ORDER_DEPENDENCY']
for x in test_seq:
if isinstance(x, str):
# Clear because of order dep injection
# In real world, this is not shared over instances
if 'require' in self.env:
del self.env['require']
argv = ['__planet', x]
emu = emulator.Emulator(argv, env=self.env)
emu.run()
elif isinstance(x, bool):
if x:
self.env['CDIST_ORDER_DEPENDENCY'] = 'on'
elif 'CDIST_ORDER_DEPENDENCY' in self.env:
del self.env['CDIST_ORDER_DEPENDENCY']
cdist_type = core.CdistType(self.local.type_path, '__planet')
for x in test_seq:
if isinstance(x, str):
obj = core.CdistObject(cdist_type, self.local.object_path,
self.local.object_marker_name, x)
reqs = set(obj.requirements)
if x in expected_requirements:
self.assertEqual(reqs, expected_requirements[x])
else:
self.assertTrue(len(reqs) == 0)
# if we get here all is fine
class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):