From da274e5ef3e8c1eb47bb9fde003248711439fc8d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 23 Nov 2019 00:25:51 +0100 Subject: [PATCH] Redefine/reimplement CDIST_ORDER_DEPENDENCY CDIST_ORDER_DEPENDENCY now defines type order dependency context. cdist (emulator) maintains global state variables, as files, order_dep_state and typeorder_dep, and per object state variable, as file, typeorder_dep. If order_dep_state exists then this defines that order dependency is turned on. If order_dep_state does not exist then order dependency is turned off. If order dependency is on then objects created after it is turned on are recorded into: * global typeorder_dep, in case of init manifest * object's typeorder_dep, in case of type's manifest. If order dependency is on then requirement is injected, where object created before current, is read from: * global typeorder_dep, in case of init manifest * object's typeorder_dep, in case of type's manifest. Every time order dependency is turned off, typeorder_dep files are removed, which means that type order list is cleared, context is cleaned. In the end cdist cleans after itself, i.e. mentioned files are removed. When running type manifest is finished typeorder_dep file is removed. When running config finishes global typeorder_dep and order_dep_state files are removed. Global type order recording is untouched. Furthermore, for completeness, type order is now recorded for each object too. --- cdist/config.py | 4 ++ cdist/core/cdist_object.py | 10 +++ cdist/core/manifest.py | 14 ++++ cdist/emulator.py | 124 +++++++++++++++++++++++--------- cdist/test/emulator/__init__.py | 43 +++++++++-- 5 files changed, 157 insertions(+), 38 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 6c30db2e..97cc1da6 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -124,6 +124,7 @@ class Config(object): """Remove files and directories for the run""" if self.remove_remote_files_dirs: self._remove_remote_files_dirs() + self.manifest.cleanup() @staticmethod def hosts(source): @@ -787,6 +788,9 @@ class Config(object): self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) try: self.manifest.run_type_manifest(cdist_object) + self.log.trace("[ORDER_DEP] Removing order dep files for %s", + cdist_object) + cdist_object.cleanup() cdist_object.state = core.CdistObject.STATE_PREPARED except cdist.Error as e: raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index 237f0ddd..114a47e0 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -243,6 +243,16 @@ class CdistObject(object): lambda obj: os.path.join(obj.base_path, obj.code_local_path)) code_remote = fsproperty.FileStringProperty( lambda obj: os.path.join(obj.base_path, obj.code_remote_path)) + typeorder = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'typeorder')) + typeorder_dep = fsproperty.FileListProperty( + lambda obj: os.path.join(obj.absolute_path, 'typeorder_dep')) + + def cleanup(self): + try: + os.remove(os.path.join(self.absolute_path, 'typeorder_dep')) + except FileNotFoundError: + pass @property def exists(self): diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 07af0ef8..8aeaf860 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -96,6 +96,10 @@ class Manifest(object): """Executes cdist manifests. """ + + ORDER_DEP_STATE_NAME = 'order_dep_state' + TYPEORDER_DEP_NAME = 'typeorder_dep' + def __init__(self, target_host, local, dry_run=False): self.target_host = target_host self.local = local @@ -212,3 +216,13 @@ class Manifest(object): type_manifest, env=self.env_type_manifest(cdist_object), message_prefix=message_prefix) + + def cleanup(self): + def _rm_file(fname): + try: + self.log.trace("[ORDER_DEP] Removing %s", fname) + os.remove(os.path.join(self.local.base_path, fname)) + except FileNotFoundError: + pass + _rm_file(Manifest.ORDER_DEP_STATE_NAME) + _rm_file(Manifest.TYPEORDER_DEP_NAME) diff --git a/cdist/emulator.py b/cdist/emulator.py index 417f2cdd..4800e2a3 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -29,6 +29,7 @@ import sys import cdist from cdist import core from cdist import flock +from cdist.core.manifest import Manifest class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -82,6 +83,11 @@ class Emulator(object): self.object_base_path = os.path.join(self.global_path, "object") self.typeorder_path = os.path.join(self.global_path, "typeorder") + self.typeorder_dep_path = os.path.join(self.global_path, + Manifest.TYPEORDER_DEP_NAME) + self.order_dep_state_path = os.path.join(self.global_path, + Manifest.ORDER_DEP_STATE_NAME) + self.type_name = os.path.basename(argv[0]) self.cdist_type = core.CdistType(self.type_base_path, self.type_name) @@ -206,6 +212,14 @@ class Emulator(object): return params def setup_object(self): + # CDIST_ORDER_DEPENDENCY state + order_dep_on = self._order_dep_on() + order_dep_defined = "CDIST_ORDER_DEPENDENCY" in self.env + if not order_dep_defined and order_dep_on: + self._set_order_dep_state_off() + if order_dep_defined and not order_dep_on: + self._set_order_dep_state_on() + # Create object with given parameters self.parameters = {} for key, value in vars(self.args).items(): @@ -237,6 +251,20 @@ class Emulator(object): # record the created object in typeorder file with open(self.typeorder_path, 'a') as typeorderfile: print(self.cdist_object.name, file=typeorderfile) + # record the created object in parent object typeorder file + __object_name = self.env.get('__object_name', None) + depname = self.cdist_object.name + if __object_name: + parent = self.cdist_object.object_from_name(__object_name) + parent.typeorder.append(self.cdist_object.name) + if self._order_dep_on(): + self.log.trace(('[ORDER_DEP] Adding %s to typeorder dep' + ' for %s'), depname, parent.name) + parent.typeorder_dep.append(depname) + elif self._order_dep_on(): + self.log.trace('[ORDER_DEP] Adding %s to global typeorder dep', + depname) + self._add_typeorder_dep(depname) # Record / Append source self.cdist_object.source.append(self.object_source) @@ -293,45 +321,73 @@ class Emulator(object): return cdist_object.name + def _order_dep_on(self): + return os.path.exists(self.order_dep_state_path) + + def _set_order_dep_state_on(self): + self.log.trace('[ORDER_DEP] Setting order dep state on') + with open(self.order_dep_state_path, 'w'): + pass + + def _set_order_dep_state_off(self): + self.log.trace('[ORDER_DEP] Setting order dep state off') + # remove order dep state file + try: + os.remove(self.order_dep_state_path) + except FileNotFoundError: + pass + # remove typeorder dep file + try: + os.remove(self.typeorder_dep_path) + except FileNotFoundError: + pass + + def _add_typeorder_dep(self, name): + with open(self.typeorder_dep_path, 'a') as f: + print(name, file=f) + + def _read_typeorder_dep(self): + try: + with open(self.typeorder_dep_path, 'r') as f: + return f.readlines() + except FileNotFoundError: + return [] + def record_requirements(self): """Record requirements.""" + order_dep_on = self._order_dep_on() + # Inject the predecessor, but not if its an override # (this would leed to an circular dependency) - if ("CDIST_ORDER_DEPENDENCY" in self.env and - 'CDIST_OVERRIDE' not in self.env): - # load object name created befor this one from typeorder file ... - with open(self.typeorder_path, 'r') as typecreationfile: - typecreationorder = typecreationfile.readlines() - # get the type created before this one ... - try: - lastcreatedtype = typecreationorder[-2].strip() - # __object_name is the name of the object whose type - # manifest is currently executed - __object_name = self.env.get('__object_name', None) - if lastcreatedtype == __object_name: - self.log.debug(("Not injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s," - " %s's type manifest is currently" - " being executed"), - lastcreatedtype, - self.cdist_object.name, - lastcreatedtype) - else: - if 'require' in self.env: - appendix = " " + lastcreatedtype - if appendix not in self.env['require']: - self.env['require'] += appendix - else: - self.env['require'] = lastcreatedtype - self.log.debug(("Injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s"), - lastcreatedtype, - self.cdist_object.name) - except IndexError: - # if no second last line, we are on the first type, - # so do not set a requirement - pass + if (order_dep_on and 'CDIST_OVERRIDE' not in self.env): + try: + # __object_name is the name of the object whose type + # manifest is currently executed + __object_name = self.env.get('__object_name', None) + # load object name created befor this one from typeorder + # dep file + if __object_name: + parent = self.cdist_object.object_from_name( + __object_name) + typeorder = parent.typeorder_dep + else: + typeorder = self._read_typeorder_dep() + # get the type created before this one + lastcreatedtype = typeorder[-2].strip() + if 'require' in self.env: + if lastcreatedtype not in self.env['require']: + self.env['require'] += " " + lastcreatedtype + else: + self.env['require'] = lastcreatedtype + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, + self.cdist_object.name) + except IndexError: + # if no second last line, we are on the first type, + # so do not set a requirement + pass reqs = set() if "require" in self.env: diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 5691093c..e375676c 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -24,8 +24,6 @@ import io import os import shutil -import string -import filecmp import random import logging @@ -34,7 +32,6 @@ from cdist import test from cdist.exec import local from cdist import emulator from cdist import core -from cdist import config import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -115,7 +112,7 @@ class EmulatorTestCase(test.CdistTestCase): def test_requirement_pattern(self): argv = ['__file', '/tmp/foobar'] self.env['require'] = '__file/etc/*' - emu = emulator.Emulator(argv, env=self.env) + emulator.Emulator(argv, env=self.env) # if we get here all is fine def test_loglevel(self): @@ -172,6 +169,44 @@ class EmulatorTestCase(test.CdistTestCase): self.assertEqual(list(file_object.requirements), ['__planet/mars']) # if we get here all is fine + def test_order_dependency_context(self): + test_seq = ('A', True, 'B', 'C', 'D', False, 'E', 'F', True, 'G', + 'H', False, 'I', ) + expected_requirements = { + 'C': set(('__planet/B', )), + 'D': set(('__planet/C', )), + 'H': set(('__planet/G', )), + } + # Ensure env var is not in env + if 'CDIST_ORDER_DEPENDENCY' in self.env: + del self.env['CDIST_ORDER_DEPENDENCY'] + + for x in test_seq: + if isinstance(x, str): + # Clear because of order dep injection + # In real world, this is not shared over instances + if 'require' in self.env: + del self.env['require'] + argv = ['__planet', x] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + elif isinstance(x, bool): + if x: + self.env['CDIST_ORDER_DEPENDENCY'] = 'on' + elif 'CDIST_ORDER_DEPENDENCY' in self.env: + del self.env['CDIST_ORDER_DEPENDENCY'] + cdist_type = core.CdistType(self.local.type_path, '__planet') + for x in test_seq: + if isinstance(x, str): + obj = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, x) + reqs = set(obj.requirements) + if x in expected_requirements: + self.assertEqual(reqs, expected_requirements[x]) + else: + self.assertTrue(len(reqs) == 0) + # if we get here all is fine + class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):