From c363fc24deb63b9ffc029af2a434bd701e045ec1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 6 May 2014 00:28:18 +0200 Subject: [PATCH 1/7] begin to make OBJECT_MARKER dynamic Signed-off-by: Nico Schottelius --- cdist/config.py | 2 +- cdist/core/__init__.py | 2 +- cdist/core/cdist_object.py | 15 +++++++-------- docs/dev/logs/2014-05-06.object-marker | 14 ++++++++++++++ 4 files changed, 23 insertions(+), 10 deletions(-) create mode 100644 docs/dev/logs/2014-05-06.object-marker diff --git a/cdist/config.py b/cdist/config.py index 73ba4710..2fd80db0 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2014 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index 66ee00a5..5dafd061 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2014 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -23,7 +24,6 @@ from cdist.core.cdist_type import CdistType from cdist.core.cdist_type import NoSuchTypeError from cdist.core.cdist_object import CdistObject from cdist.core.cdist_object import IllegalObjectIdError -from cdist.core.cdist_object import OBJECT_MARKER from cdist.core.explorer import Explorer from cdist.core.manifest import Manifest from cdist.core.code import Code diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index e8c58a67..d13c33dd 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -32,9 +32,6 @@ from cdist.util import fsproperty log = logging.getLogger(__name__) -OBJECT_MARKER = '.cdist' - - class IllegalObjectIdError(cdist.Error): def __init__(self, object_id, message=None): self.object_id = object_id @@ -66,16 +63,18 @@ class CdistObject(object): STATE_RUNNING = "running" STATE_DONE = "done" - def __init__(self, cdist_type, base_path, object_id=''): + def __init__(self, cdist_type, base_path, object_marker=".cdist", object_id=''): self.cdist_type = cdist_type # instance of Type self.base_path = base_path self.object_id = object_id + self.object_marker = object_marker + self.validate_object_id() self.sanitise_object_id() self.name = self.join_name(self.cdist_type.name, self.object_id) - self.path = os.path.join(self.cdist_type.path, self.object_id, OBJECT_MARKER) + self.path = os.path.join(self.cdist_type.path, self.object_id, self.object_marker) self.absolute_path = os.path.join(self.base_path, self.path) self.code_local_path = os.path.join(self.path, "code-local") self.code_remote_path = os.path.join(self.path, "code-remote") @@ -97,7 +96,7 @@ class CdistObject(object): def list_object_names(cls, object_base_path): """Return a list of object names""" for path, dirs, files in os.walk(object_base_path): - if OBJECT_MARKER in dirs: + if self.object_marker in dirs: yield os.path.relpath(path, object_base_path) @staticmethod @@ -127,8 +126,8 @@ class CdistObject(object): """Validate the given object_id and raise IllegalObjectIdError if it's not valid. """ if self.object_id: - if OBJECT_MARKER in self.object_id.split(os.sep): - raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % OBJECT_MARKER) + if self.object_marker in self.object_id.split(os.sep): + raise IllegalObjectIdError(self.object_id, 'object_id may not contain \'%s\'' % self.object_marker) if '//' in self.object_id: raise IllegalObjectIdError(self.object_id, 'object_id may not contain //') if self.object_id == '.': diff --git a/docs/dev/logs/2014-05-06.object-marker b/docs/dev/logs/2014-05-06.object-marker new file mode 100644 index 00000000..4878a2f5 --- /dev/null +++ b/docs/dev/logs/2014-05-06.object-marker @@ -0,0 +1,14 @@ +Change object marker from .cdist to .cdist-TEMPNAME to allow using +object ids that contain / are .cdist. + +Changes required: + + cdist/emulator.py: + needs to know suffix/name + tests: + allow object id named /.cdist + tests: + many + cdist/config.py: + have suffix + From 73c77dd2d33e38f4cac6939f7a2739930db20502 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 5 Mar 2015 15:02:26 +0100 Subject: [PATCH 2/7] Let core use random .cdist directory for objects Signed-off-by: Nico Schottelius --- cdist/__init__.py | 4 +--- cdist/config.py | 5 ++-- cdist/core/__init__.py | 2 +- cdist/core/cdist_object.py | 28 +++++++++++++---------- cdist/emulator.py | 14 ++++++++---- cdist/exec/local.py | 21 +++++++++++++++-- cdist/shell.py | 2 +- docs/dev/logs/2015-02-22.allow_dot_cdist | 29 ++++++++++++++++++++++++ 8 files changed, 79 insertions(+), 26 deletions(-) create mode 100644 docs/dev/logs/2015-02-22.allow_dot_cdist diff --git a/cdist/__init__.py b/cdist/__init__.py index 20c76b31..4454a3ac 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -41,8 +41,6 @@ BANNER = """ "P' "" "" """ -DOT_CDIST = ".cdist" - REMOTE_COPY = "scp -o User=root -q" REMOTE_EXEC = "ssh -o User=root -q" diff --git a/cdist/config.py b/cdist/config.py index a0c787ac..bba9bfcb 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -162,7 +162,8 @@ class Config(object): def object_list(self): """Short name for object list retrieval""" for cdist_object in core.CdistObject.list_objects(self.local.object_path, - self.local.type_path): + self.local.type_path, + self.local.object_marker_name): if cdist_object.cdist_type.is_install: self.log.debug("Running in config mode, ignoring install object: {0}".format(cdist_object)) else: diff --git a/cdist/core/__init__.py b/cdist/core/__init__.py index 5dafd061..d773fc01 100644 --- a/cdist/core/__init__.py +++ b/cdist/core/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2014-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index d13c33dd..ba0fdf11 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Daniel Heule (hda at sfs.biz) # # This file is part of cdist. @@ -63,7 +63,7 @@ class CdistObject(object): STATE_RUNNING = "running" STATE_DONE = "done" - def __init__(self, cdist_type, base_path, object_marker=".cdist", object_id=''): + def __init__(self, cdist_type, base_path, object_marker, object_id): self.cdist_type = cdist_type # instance of Type self.base_path = base_path self.object_id = object_id @@ -75,30 +75,34 @@ class CdistObject(object): self.name = self.join_name(self.cdist_type.name, self.object_id) self.path = os.path.join(self.cdist_type.path, self.object_id, self.object_marker) + self.absolute_path = os.path.join(self.base_path, self.path) self.code_local_path = os.path.join(self.path, "code-local") self.code_remote_path = os.path.join(self.path, "code-remote") self.parameter_path = os.path.join(self.path, "parameter") @classmethod - def list_objects(cls, object_base_path, type_base_path): + def list_objects(cls, object_base_path, type_base_path, object_marker): """Return a list of object instances""" - for object_name in cls.list_object_names(object_base_path): + for object_name in cls.list_object_names(object_base_path, object_marker): type_name, object_id = cls.split_name(object_name) - yield cls(cdist.core.CdistType(type_base_path, type_name), object_base_path, object_id=object_id) + yield cls(cdist.core.CdistType(type_base_path, type_name), + base_path=object_base_path, + object_marker=object_marker, + object_id=object_id) + + @classmethod + def list_object_names(cls, object_base_path, object_marker): + """Return a list of object names""" + for path, dirs, files in os.walk(object_base_path): + if object_marker in dirs: + yield os.path.relpath(path, object_base_path) @classmethod def list_type_names(cls, object_base_path): """Return a list of type names""" return os.listdir(object_base_path) - @classmethod - def list_object_names(cls, object_base_path): - """Return a list of object names""" - for path, dirs, files in os.walk(object_base_path): - if self.object_marker in dirs: - yield os.path.relpath(path, object_base_path) - @staticmethod def split_name(object_name): """split_name('__type_name/the/object_id') -> ('__type_name', 'the/object_id') diff --git a/cdist/emulator.py b/cdist/emulator.py index 41834fbf..abd8c884 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # 2014 Daniel Heule (hda at sfs.biz) # @@ -64,9 +64,10 @@ class Emulator(object): self.global_path = self.env['__global'] self.target_host = self.env['__target_host'] - # Internally only + # Internal variables self.object_source = self.env['__cdist_manifest'] self.type_base_path = self.env['__cdist_type_base_path'] + self.object_marker = self.env['__cdist_object_marker'] except KeyError as e: raise MissingRequiredEnvironmentVariableError(e.args[0]) @@ -131,13 +132,16 @@ class Emulator(object): self.log.debug('Args: %s' % self.args) def setup_object(self): - # Setup object_id - FIXME: unset / do not setup anymore! - if not self.cdist_type.is_singleton: + # Setup object - and ensure it is not in args + if self.cdist_type.is_singleton: + self.object_id = False + else: self.object_id = self.args.object_id[0] del self.args.object_id # Instantiate the cdist object we are defining - self.cdist_object = core.CdistObject(self.cdist_type, self.object_base_path, self.object_id) + self.cdist_object = core.CdistObject(self.cdist_type, + self.object_base_path, self.object_marker, self.object_id) # Create object with given parameters self.parameters = {} diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 2f75ffd4..40f34e25 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -66,9 +66,9 @@ class Local(object): self._init_log() self._init_permissions() self._init_paths() + self._init_object_marker() self._init_conf_dirs() - @property def dist_conf_dir(self): return os.path.abspath(os.path.join(os.path.dirname(cdist.__file__), "conf")) @@ -103,6 +103,12 @@ class Local(object): self.type_path = os.path.join(self.conf_path, "type") + def _init_object_marker(self): + self.object_marker_file = os.path.join(self.base_path, "object_marker") + + # Does not need to be secure - just randomly different from .cdist + self.object_marker_name = tempfile.mktemp(prefix='.cdist-', dir='') + def _init_conf_dirs(self): self.conf_dirs = [] @@ -125,6 +131,7 @@ class Local(object): def _init_directories(self): self.mkdir(self.conf_path) self.mkdir(self.global_explorer_out_path) + self.mkdir(self.object_path) self.mkdir(self.bin_path) def create_files_dirs(self): @@ -132,6 +139,13 @@ class Local(object): self._create_conf_path_and_link_conf_dirs() self._create_messages() self._link_types_for_emulator() + self._setup_object_marker_file() + + def _setup_object_marker_file(self): + with open(self.object_marker_file, 'w') as fd: + fd.write("%s\n" % self.object_marker_name) + + self.log.debug("Object marker %s saved in %s" % (self.object_marker_name, self.object_marker_file)) def _init_cache_dir(self, cache_dir): @@ -166,6 +180,9 @@ class Local(object): # Export __target_host for use in __remote_{copy,exec} scripts env['__target_host'] = self.target_host + # Export for emulator + env['__cdist_object_marker'] = self.object_marker_name + if message_prefix: message = cdist.message.Message(message_prefix, self.messages_path) env.update(message.env) diff --git a/cdist/shell.py b/cdist/shell.py index 8ca68115..d0921bc9 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2013-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # diff --git a/docs/dev/logs/2015-02-22.allow_dot_cdist b/docs/dev/logs/2015-02-22.allow_dot_cdist new file mode 100644 index 00000000..58a1b041 --- /dev/null +++ b/docs/dev/logs/2015-02-22.allow_dot_cdist @@ -0,0 +1,29 @@ +- locate code that references .cdist + - cdist_object.py +- need to change code that handles objects? + - setup object marker + exec/local.py + - cdist/emulator.py + - need to know the marker name + - shell.py + - test/manifest/__init__.py + - core/code.py: + - core/manifest.py: + - core/manifest.py: + - list_object_names() needs to know the marker -- used BY: + - list_objects + - cdist/test/cdist_object/__init__.py + - cdist/config.py + - cdist/test/cdist_object/__init__.py + + - list_object_names + - needs to have object_marker + +- or modify object code to load name +- setup a per-run random name + - local.py +- use the per-run random name +- create test + + def __init__(self, cdist_type, base_path, object_marker=".cdist", object_id=''): + From fa6e389fdd7da946a633a4b3bfa46361adc44129 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 5 Mar 2015 15:23:53 +0100 Subject: [PATCH 3/7] cannot use False for object id, as it is being used for os.path.join() in object Signed-off-by: Nico Schottelius --- cdist/emulator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/emulator.py b/cdist/emulator.py index abd8c884..3f553412 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -134,7 +134,7 @@ class Emulator(object): def setup_object(self): # Setup object - and ensure it is not in args if self.cdist_type.is_singleton: - self.object_id = False + self.object_id = '' else: self.object_id = self.args.object_id[0] del self.args.object_id From d08c29b581545ce5751c6ed7d579193c873e589b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 5 Mar 2015 18:32:47 +0100 Subject: [PATCH 4/7] fix most test cases broken by no-dot-cdist change Signed-off-by: Nico Schottelius --- cdist/core/cdist_object.py | 3 +- cdist/test/cdist_object/__init__.py | 98 +++++++++++++------ .../fixtures/object/__first/.keep | 0 .../object/__first/child/.cdist/.keep | 0 .../fixtures/object/__first/dog/.cdist/.keep | 0 .../fixtures/object/__first/man/.cdist/.keep | 0 .../object/__first/woman/.cdist/.keep | 0 .../fixtures/object/__second/.keep | 0 .../object/__second/on-the/.cdist/.keep | 0 .../object/__second/under-the/.cdist/.keep | 0 .../fixtures/object/__third/.keep | 0 .../fixtures/object/__third/moon/.cdist/.keep | 0 .../object/__third/moon/.cdist/parameter/name | 1 - .../__third/moon/.cdist/parameter/planet | 1 - cdist/test/code/__init__.py | 4 +- cdist/test/config/__init__.py | 4 +- cdist/test/emulator/__init__.py | 31 +++--- cdist/test/explorer/__init__.py | 6 +- cdist/test/manifest/__init__.py | 2 +- 19 files changed, 97 insertions(+), 53 deletions(-) delete mode 100644 cdist/test/cdist_object/fixtures/object/__first/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__first/child/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__first/dog/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__first/man/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__first/woman/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__second/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__second/on-the/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__second/under-the/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__third/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/.keep delete mode 100644 cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/name delete mode 100644 cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/planet diff --git a/cdist/core/cdist_object.py b/cdist/core/cdist_object.py index ba0fdf11..8c6ee9c9 100644 --- a/cdist/core/cdist_object.py +++ b/cdist/core/cdist_object.py @@ -158,12 +158,13 @@ class CdistObject(object): base_path = self.base_path type_path = self.cdist_type.base_path + object_marker = self.object_marker type_name, object_id = self.split_name(object_name) cdist_type = self.cdist_type.__class__(type_path, type_name) - return self.__class__(cdist_type, base_path, object_id=object_id) + return self.__class__(cdist_type, base_path, object_marker, object_id=object_id) def __repr__(self): return '' % self.name diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index 3c25a959..9c075acb 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Daniel Heule (hda at sfs.biz) # # This file is part of cdist. @@ -23,6 +23,7 @@ import os import shutil +import tempfile from cdist import test from cdist import core @@ -32,37 +33,48 @@ import cdist import os.path as op my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') -object_base_path = op.join(fixtures, 'object') type_base_path = op.join(fixtures, 'type') +OBJECT_MARKER_NAME = '.cdist-pseudo-random' + +expected_object_names = sorted([ + '__first/child', + '__first/dog', + '__first/man', + '__first/woman', + '__second/on-the', + '__second/under-the', + '__third/moon']) + + class ObjectClassTestCase(test.CdistTestCase): def setUp(self): - self.expected_object_names = sorted([ - '__first/child', - '__first/dog', - '__first/man', - '__first/woman', - '__second/on-the', - '__second/under-the', - '__third/moon']) + + self.tempdir = tempfile.mkdtemp(prefix="cdist.test") + self.object_base_path = self.tempdir self.expected_objects = [] - for cdist_object_name in self.expected_object_names: + for cdist_object_name in expected_object_names: cdist_type, cdist_object_id = cdist_object_name.split("/", 1) - cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), object_base_path, cdist_object_id) + cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), self.object_base_path, + OBJECT_MARKER_NAME, cdist_object_id) + cdist_object.create() self.expected_objects.append(cdist_object) + def tearDown(self): + shutil.rmtree(self.tempdir) + def test_list_object_names(self): - found_object_names = sorted(list(core.CdistObject.list_object_names(object_base_path))) - self.assertEqual(found_object_names, self.expected_object_names) + found_object_names = sorted(list(core.CdistObject.list_object_names(self.object_base_path, OBJECT_MARKER_NAME))) + self.assertEqual(found_object_names, expected_object_names) def test_list_type_names(self): - type_names = list(cdist.core.CdistObject.list_type_names(object_base_path)) + type_names = list(cdist.core.CdistObject.list_type_names(self.object_base_path)) self.assertEqual(sorted(type_names), ['__first', '__second', '__third']) def test_list_objects(self): - found_objects = sorted(list(core.CdistObject.list_objects(object_base_path, type_base_path))) + found_objects = sorted(list(core.CdistObject.list_objects(self.object_base_path, type_base_path, OBJECT_MARKER_NAME))) self.assertEqual(found_objects, self.expected_objects) def test_create_singleton(self): @@ -77,41 +89,65 @@ class ObjectClassTestCase(test.CdistTestCase): self.expected_objects[0].object_from_name("__first") class ObjectIdTestCase(test.CdistTestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp(prefix="cdist.test") + self.object_base_path = self.tempdir + + self.expected_objects = [] + for cdist_object_name in expected_object_names: + cdist_type, cdist_object_id = cdist_object_name.split("/", 1) + cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), self.object_base_path, + OBJECT_MARKER_NAME, cdist_object_id) + cdist_object.create() + self.expected_objects.append(cdist_object) + + def tearDown(self): + shutil.rmtree(self.tempdir) + def test_object_id_contains_double_slash(self): cdist_type = core.CdistType(type_base_path, '__third') illegal_object_id = '/object_id//may/not/contain/double/slash' with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % core.OBJECT_MARKER + illegal_object_id = 'object_id/may/not/contain/%s/anywhere' % OBJECT_MARKER_NAME with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_contains_object_marker_string(self): cdist_type = core.CdistType(type_base_path, '__third') - illegal_object_id = 'object_id/may/contain_%s_in_filename' % core.OBJECT_MARKER - core.CdistObject(cdist_type, object_base_path, illegal_object_id) + illegal_object_id = 'object_id/may/contain_%s_in_filename' % OBJECT_MARKER_NAME + core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) # if we get here, the test passed def test_object_id_contains_only_dot(self): cdist_type = core.CdistType(type_base_path, '__third') illegal_object_id = '.' with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) def test_object_id_on_singleton_type(self): cdist_type = core.CdistType(type_base_path, '__test_singleton') illegal_object_id = 'object_id' with self.assertRaises(core.IllegalObjectIdError): - core.CdistObject(cdist_type, object_base_path, illegal_object_id) + core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) class ObjectTestCase(test.CdistTestCase): def setUp(self): + self.tempdir = tempfile.mkdtemp(prefix="cdist.test") + self.object_base_path = self.tempdir + self.cdist_type = core.CdistType(type_base_path, '__third') - self.cdist_object = core.CdistObject(self.cdist_type, object_base_path, 'moon') + self.cdist_object = core.CdistObject(self.cdist_type, self.object_base_path, OBJECT_MARKER_NAME, 'moon') + self.cdist_object.create() + + self.cdist_object.parameters['planet'] = 'Saturn' + self.cdist_object.parameters['name'] = 'Prometheus' + def tearDown(self): self.cdist_object.prepared = False @@ -121,6 +157,8 @@ class ObjectTestCase(test.CdistTestCase): self.cdist_object.code_remote = '' self.cdist_object.state = '' + shutil.rmtree(self.tempdir) + def test_name(self): self.assertEqual(self.cdist_object.name, '__third/moon') @@ -128,22 +166,22 @@ class ObjectTestCase(test.CdistTestCase): self.assertEqual(self.cdist_object.object_id, 'moon') def test_path(self): - self.assertEqual(self.cdist_object.path, '__third/moon/.cdist') + self.assertEqual(self.cdist_object.path, "__third/moon/%s" % OBJECT_MARKER_NAME) def test_absolute_path(self): - self.assertEqual(self.cdist_object.absolute_path, os.path.join(object_base_path, '__third/moon/.cdist')) + self.assertEqual(self.cdist_object.absolute_path, os.path.join(self.object_base_path, "__third/moon/%s" % OBJECT_MARKER_NAME)) def test_code_local_path(self): - self.assertEqual(self.cdist_object.code_local_path, '__third/moon/.cdist/code-local') + self.assertEqual(self.cdist_object.code_local_path, "__third/moon/%s/code-local" % OBJECT_MARKER_NAME) def test_code_remote_path(self): - self.assertEqual(self.cdist_object.code_remote_path, '__third/moon/.cdist/code-remote') + self.assertEqual(self.cdist_object.code_remote_path, "__third/moon/%s/code-remote" % OBJECT_MARKER_NAME) def test_parameter_path(self): - self.assertEqual(self.cdist_object.parameter_path, '__third/moon/.cdist/parameter') + self.assertEqual(self.cdist_object.parameter_path, "__third/moon/%s/parameter" % OBJECT_MARKER_NAME) def test_explorer_path(self): - self.assertEqual(self.cdist_object.explorer_path, '__third/moon/.cdist/explorer') + self.assertEqual(self.cdist_object.explorer_path, "__third/moon/%s/explorer" % OBJECT_MARKER_NAME) def test_parameters(self): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} diff --git a/cdist/test/cdist_object/fixtures/object/__first/.keep b/cdist/test/cdist_object/fixtures/object/__first/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__first/child/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__first/child/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__first/dog/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__first/dog/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__first/man/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__first/man/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__first/woman/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__first/woman/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__second/.keep b/cdist/test/cdist_object/fixtures/object/__second/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__second/on-the/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__second/on-the/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__second/under-the/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__second/under-the/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__third/.keep b/cdist/test/cdist_object/fixtures/object/__third/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/.keep b/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/.keep deleted file mode 100644 index e69de29b..00000000 diff --git a/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/name b/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/name deleted file mode 100644 index 4129a761..00000000 --- a/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/name +++ /dev/null @@ -1 +0,0 @@ -Prometheus diff --git a/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/planet b/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/planet deleted file mode 100644 index 8e6ee422..00000000 --- a/cdist/test/cdist_object/fixtures/object/__third/moon/.cdist/parameter/planet +++ /dev/null @@ -1 +0,0 @@ -Saturn diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 796e8a51..689fcff6 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -61,7 +61,7 @@ class CodeTestCase(test.CdistTestCase): self.code = code.Code(self.target_host, self.local, self.remote) self.cdist_type = core.CdistType(self.local.type_path, '__dump_environment') - self.cdist_object = core.CdistObject(self.cdist_type, self.local.object_path, 'whatever') + self.cdist_object = core.CdistObject(self.cdist_type, self.local.object_path, 'whatever', self.local.object_marker_name) self.cdist_object.create() def tearDown(self): diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index 70501c89..452ce042 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Daniel Heule (hda at sfs.biz) # # This file is part of cdist. @@ -67,7 +67,7 @@ class ConfigRunTestCase(test.CdistTestCase): self.config = cdist.config.Config(self.local, self.remote) - self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path)) + self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path, self.local.object_marker_name)) self.object_index = dict((o.name, o) for o in self.objects) self.object_names = [o.name for o in self.objects] diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 870d6245..62000931 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -57,10 +57,15 @@ class EmulatorTestCase(test.CdistTestCase): self.manifest = core.Manifest(self.target_host, self.local) self.env = self.manifest.env_initial_manifest(self.script) + self.env['__cdist_object_marker'] = self.local.object_marker_name def tearDown(self): shutil.rmtree(self.temp_dir) +# def test_missing_object_marker_variable(self): +# del self.env['__cdist_object_marker'] +# self.assertRaises(KeyError, emulator.Emulator, argv, env=self.env) + def test_nonexistent_type_exec(self): argv = ['__does-not-exist'] self.assertRaises(core.cdist_type.NoSuchTypeError, emulator.Emulator, argv, env=self.env) @@ -73,7 +78,7 @@ class EmulatorTestCase(test.CdistTestCase): def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] - self.env['require'] = '__file/bad/id/with/.cdist/inside' + self.env['require'] = "__file/bad/id/with/%s/inside" % self.local.object_marker_name emu = emulator.Emulator(argv, env=self.env) self.assertRaises(core.IllegalObjectIdError, emu.run) @@ -118,10 +123,10 @@ class EmulatorTestCase(test.CdistTestCase): emu.run() # now load the objects and verify the require parameter of the objects cdist_type = core.CdistType(self.local.type_path, '__planet') - erde_object = core.CdistObject(cdist_type, self.local.object_path, 'erde') - mars_object = core.CdistObject(cdist_type, self.local.object_path, 'mars') + erde_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'erde') + mars_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'mars') cdist_type = core.CdistType(self.local.type_path, '__file') - file_object = core.CdistObject(cdist_type, self.local.object_path, '/tmp/cdisttest') + file_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, '/tmp/cdisttest') # now test the recorded requirements self.assertTrue(len(erde_object.requirements) == 0) self.assertEqual(list(mars_object.requirements), ['__planet/erde']) @@ -150,7 +155,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): initial_manifest = os.path.join(self.local.manifest_path, "init") self.manifest.run_initial_manifest(initial_manifest) cdist_type = core.CdistType(self.local.type_path, '__saturn') - cdist_object = core.CdistObject(cdist_type, self.local.object_path) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, '') self.manifest.run_type_manifest(cdist_object) expected = ['__planet/Saturn', '__moon/Prometheus'] self.assertEqual(sorted(cdist_object.autorequire), sorted(expected)) @@ -172,6 +177,7 @@ class OverrideTestCase(test.CdistTestCase): self.manifest = core.Manifest(self.target_host, self.local) self.env = self.manifest.env_initial_manifest(self.script) + self.env['__cdist_object_marker'] = self.local.object_marker_name def tearDown(self): shutil.rmtree(self.temp_dir) @@ -211,6 +217,7 @@ class ArgumentsTestCase(test.CdistTestCase): self.manifest = core.Manifest(self.target_host, self.local) self.env = self.manifest.env_initial_manifest(self.script) + self.env['__cdist_object_marker'] = self.local.object_marker_name def tearDown(self): shutil.rmtree(self.temp_dir) @@ -222,7 +229,7 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, '__arguments_with_dashes') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'some-id') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'some-id') self.assertTrue('with-dash' in cdist_object.parameters) def test_boolean(self): @@ -234,7 +241,7 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) self.assertTrue('boolean1' in cdist_object.parameters) self.assertFalse('boolean2' in cdist_object.parameters) # empty file -> True @@ -242,17 +249,17 @@ class ArgumentsTestCase(test.CdistTestCase): def test_required_arguments(self): """check whether assigning required parameter works""" + type_name = '__arguments_required' object_id = 'some-id' value = 'some value' argv = [type_name, object_id, '--required1', value, '--required2', value] -# print(self.env) os.environ.update(self.env) emu = emulator.Emulator(argv) emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) self.assertTrue('required1' in cdist_object.parameters) self.assertTrue('required2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['required1'], value) @@ -278,7 +285,7 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) self.assertTrue('optional1' in cdist_object.parameters) self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) @@ -293,7 +300,7 @@ class ArgumentsTestCase(test.CdistTestCase): emu.run() cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) self.assertTrue('optional1' in cdist_object.parameters) self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) @@ -346,7 +353,7 @@ class StdinTestCase(test.CdistTestCase): ###################################################################### # Create path where stdin should reside at cdist_type = core.CdistType(self.local.type_path, type_name) - cdist_object = core.CdistObject(cdist_type, self.local.object_path, object_id) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, object_id) stdin_out_path = os.path.join(cdist_object.absolute_path, 'stdin') ###################################################################### diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py index 92ef75a3..335d0e32 100644 --- a/cdist/test/explorer/__init__.py +++ b/cdist/test/explorer/__init__.py @@ -127,7 +127,7 @@ class ExplorerClassTestCase(test.CdistTestCase): def test_transfer_object_parameters(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') cdist_object.create() cdist_object.parameters = {'first': 'first value', 'second': 'second value'} self.explorer.transfer_object_parameters(cdist_object) @@ -137,14 +137,14 @@ class ExplorerClassTestCase(test.CdistTestCase): def test_run_type_explorer(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') self.explorer.transfer_type_explorers(cdist_type) output = self.explorer.run_type_explorer('world', cdist_object) self.assertEqual(output, 'hello\n') def test_run_type_explorers(self): cdist_type = core.CdistType(self.local.type_path, '__test_type') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') cdist_object.create() self.explorer.run_type_explorers(cdist_object) self.assertEqual(cdist_object.explorers, {'world': 'hello'}) diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index c375a80f..cc60c844 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -84,7 +84,7 @@ class ManifestTestCase(test.CdistTestCase): def test_type_manifest_environment(self): cdist_type = core.CdistType(self.local.type_path, '__dump_environment') - cdist_object = core.CdistObject(cdist_type, self.local.object_path, 'whatever') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') handle, output_file = self.mkstemp(dir=self.temp_dir) os.close(handle) os.environ['__cdist_test_out'] = output_file From b51e9daccc993532fee6619b53f0ade987410ea0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 5 Mar 2015 18:51:25 +0100 Subject: [PATCH 5/7] fix all tests besides one Signed-off-by: Nico Schottelius --- cdist/test/config/__init__.py | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index 452ce042..a36567de 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -23,6 +23,7 @@ import os import shutil +import tempfile from cdist import test from cdist import core @@ -35,10 +36,14 @@ import cdist.core.cdist_object import os.path as op my_dir = op.abspath(op.dirname(__file__)) fixtures = op.join(my_dir, 'fixtures') -object_base_path = op.join(fixtures, 'object') type_base_path = op.join(fixtures, 'type') add_conf_dir = op.join(fixtures, 'conf') +expected_object_names = sorted([ + '__first/man', + '__second/on-the', + '__third/moon']) + class ConfigRunTestCase(test.CdistTestCase): def setUp(self): @@ -54,6 +59,20 @@ class ConfigRunTestCase(test.CdistTestCase): target_host=self.target_host, base_path=self.local_dir) + # Setup test objects + self.object_base_path = op.join(self.temp_dir, 'object') + + self.objects = [] + for cdist_object_name in expected_object_names: + cdist_type, cdist_object_id = cdist_object_name.split("/", 1) + cdist_object = core.CdistObject(core.CdistType(type_base_path, cdist_type), self.object_base_path, + self.local.object_marker_name, cdist_object_id) + cdist_object.create() + self.objects.append(cdist_object) + + self.object_index = dict((o.name, o) for o in self.objects) + self.object_names = [o.name for o in self.objects] + self.remote_dir = os.path.join(self.temp_dir, "remote") os.mkdir(self.remote_dir) self.remote = cdist.exec.remote.Remote( @@ -62,15 +81,11 @@ class ConfigRunTestCase(test.CdistTestCase): remote_exec=self.remote_exec, base_path=self.remote_dir) - self.local.object_path = object_base_path + self.local.object_path = self.object_base_path self.local.type_path = type_base_path self.config = cdist.config.Config(self.local, self.remote) - self.objects = list(core.CdistObject.list_objects(object_base_path, type_base_path, self.local.object_marker_name)) - self.object_index = dict((o.name, o) for o in self.objects) - self.object_names = [o.name for o in self.objects] - def tearDown(self): for o in self.objects: o.requirements = [] From b0321895a7e9810c8f9b5fab6016a835aaed511e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 5 Mar 2015 19:20:06 +0100 Subject: [PATCH 6/7] fix all tests Signed-off-by: Nico Schottelius --- cdist/test/emulator/__init__.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 62000931..f90e5320 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Daniel Heule (hda at sfs.biz) # # This file is part of cdist. @@ -323,10 +323,6 @@ class StdinTestCase(test.CdistTestCase): self.local.create_files_dirs() - self.manifest = core.Manifest( - target_host=self.target_host, - local = self.local) - def tearDown(self): os.environ = self.orig_environ shutil.rmtree(self.temp_dir) @@ -347,8 +343,11 @@ class StdinTestCase(test.CdistTestCase): object_id = "cdist-test-id" argv = [type_name, object_id] - initial_manifest_path = "/cdist-test/path/that/does/not/exist" - env = self.manifest.env_initial_manifest(initial_manifest_path) + env = os.environ.copy() + env['__cdist_manifest'] = "/cdist-test/path/that/does/not/exist" + env['__cdist_object_marker'] = self.local.object_marker_name + env['__cdist_type_base_path'] = self.local.type_path + env['__global'] = self.local.base_path ###################################################################### # Create path where stdin should reside at From e225ec3f77b483c0d5238449ff84a7424e7fbf2b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 5 Mar 2015 19:45:41 +0100 Subject: [PATCH 7/7] ++changes Signed-off-by: Nico Schottelius --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index f55a01ac..31a23505 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * New type: __firewalld_rule (Nico Schottelius) + * Type __consul_agent: add support for acl options (Steven Armstrong) + * Core: Support object ids '.cdist' (Nico Schottelius) 3.1.11: 2015-02-27 * New type: __staged_file: Manage staged files (Steven Armstrong)