From d7adff4e21b2d5655925603b8123b274dc1ec123 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 14:21:47 +0200 Subject: [PATCH 1/9] silently ignore error when deleting nonexistent file Signed-off-by: Steven Armstrong --- lib/cdist/util/fsproperty.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/cdist/util/fsproperty.py b/lib/cdist/util/fsproperty.py index c18a1553..68d5edaa 100644 --- a/lib/cdist/util/fsproperty.py +++ b/lib/cdist/util/fsproperty.py @@ -134,7 +134,11 @@ class FileListProperty(FileList): def __set__(self, obj, value): self._set_path(obj) - os.unlink(self.path) + try: + os.unlink(self.path) + except EnvironmentError: + # ignored + pass for item in value: self.append(item) From 45ff8f727ffd07418f5cfc646ab177bb8c95cbe8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 14:22:29 +0200 Subject: [PATCH 2/9] tests for Object source as list Signed-off-by: Steven Armstrong --- lib/cdist/test/object/__init__.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/cdist/test/object/__init__.py b/lib/cdist/test/object/__init__.py index eee1563c..decf67bf 100644 --- a/lib/cdist/test/object/__init__.py +++ b/lib/cdist/test/object/__init__.py @@ -62,7 +62,7 @@ class ObjectTestCase(unittest.TestCase): self.cdist_object.changed = False self.cdist_object.prepared = False self.cdist_object.ran = False - self.cdist_object.source = "" + self.cdist_object.source = [] def test_name(self): self.assertEqual(self.cdist_object.name, '__third/moon') @@ -118,8 +118,8 @@ class ObjectTestCase(unittest.TestCase): self.assertTrue(self.cdist_object.ran) def test_source(self): - self.assertEqual(self.cdist_object.source, '') + self.assertEqual(list(self.cdist_object.source), []) def test_source_after_changing(self): - self.cdist_object.source = '/path/to/manifest' - self.assertEqual(self.cdist_object.source, '/path/to/manifest') + self.cdist_object.source = ['/path/to/manifest'] + self.assertEqual(list(self.cdist_object.source), ['/path/to/manifest']) From 2c6c9bcab5d9f80be60ffd43357a86e79385d9e3 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 15:44:31 +0200 Subject: [PATCH 3/9] Object handles its explorers itself Signed-off-by: Steven Armstrong --- lib/cdist/core/object.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index 76634a65..b7890eeb 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -81,6 +81,7 @@ class Object(object): """define equality as 'attributes are the same'""" return self.__dict__ == other.__dict__ + # FIXME: still needed? @property def explorer_path(self): """Create and return the relative path to this objects explorers""" @@ -93,6 +94,7 @@ class Object(object): requirements = fsproperty.FileListProperty(lambda obj: os.path.join(obj.absolute_path, 'require')) parameters = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.absolute_path, 'parameter')) + explorers = fsproperty.DirectoryDictProperty(lambda obj: os.path.join(obj.base_path, obj.explorer_path)) changed = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "changed")) prepared = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "prepared")) ran = fsproperty.FileBooleanProperty(lambda obj: os.path.join(obj.absolute_path, "ran")) From d03f23129b0970719c91cb0366e64a9056b054c7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 15:45:10 +0200 Subject: [PATCH 4/9] test and fixtures for Object explorers Signed-off-by: Steven Armstrong --- lib/cdist/test/object/__init__.py | 13 +++++++++++++ .../object/__third/moon/.cdist/explorer/.keep | 0 2 files changed, 13 insertions(+) delete mode 100644 lib/cdist/test/object/fixtures/object/__third/moon/.cdist/explorer/.keep diff --git a/lib/cdist/test/object/__init__.py b/lib/cdist/test/object/__init__.py index decf67bf..6f26fd19 100644 --- a/lib/cdist/test/object/__init__.py +++ b/lib/cdist/test/object/__init__.py @@ -92,6 +92,19 @@ class ObjectTestCase(unittest.TestCase): expected_parameters = {'planet': 'Saturn', 'name': 'Prometheus'} self.assertEqual(self.cdist_object.parameters, expected_parameters) + def test_explorers(self): + self.assertEqual(self.cdist_object.explorers, {}) + + def test_explorers_after_changing(self): + expected = {'first': 'foo', 'second': 'bar'} + # when set, written to file + self.cdist_object.explorers = expected + # when accessed, read from file + self.assertEqual(self.cdist_object.explorers, expected) + # remove dynamically created folder + self.cdist_object.explorers = {} + os.rmdir(os.path.join(self.cdist_object.base_path, self.cdist_object.explorer_path)) + def test_requirements(self): expected = [] self.assertEqual(list(self.cdist_object.requirements), expected) diff --git a/lib/cdist/test/object/fixtures/object/__third/moon/.cdist/explorer/.keep b/lib/cdist/test/object/fixtures/object/__third/moon/.cdist/explorer/.keep deleted file mode 100644 index e69de29b..00000000 From e437cd90d66870e299d807442e5da66c962b2b46 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 16:06:21 +0200 Subject: [PATCH 5/9] rewrite exec Signed-off-by: Steven Armstrong --- lib/cdist/exec.py | 120 ++++++++++++++++++++++++---------------------- 1 file changed, 62 insertions(+), 58 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 7ec9bc0a..b3b86a52 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -23,75 +23,79 @@ import logging import os import subprocess -log = logging.getLogger(__name__) - import cdist -class ExecWrapper(object): - def __init__(self, remote_exec, remote_copy, target_host): +log = logging.getLogger(__name__) + + +class Wrapper(object): + def __init__(self, target_host, remote_exec, remote_copy): + self.target_host = target_host self.remote_exec = remote_exec - -def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - if remote_prefix: - remote_prefix = os.environ['__remote_exec'].split() - remote_prefix.append(os.environ['__target_host']) - args[0][:0] = remote_prefix - - log.debug("Shell exec cmd: %s", args) - - if 'env' in kargs: - log.debug("Shell exec env: %s", kargs['env']) - - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - - if remote_prefix: - run_or_fail(["cat", script], remote_prefix=remote_prefix) - - else: - try: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - except IOError as error: - raise cdist.Error(str(error)) - - raise cdist.Error("Command failed (shell): " + " ".join(*args)) - except OSError as error: - raise cdist.Error(" ".join(*args) + ": " + error.args[1]) - -def run_or_fail(*args, remote_prefix=False, **kargs): - if remote_prefix: - remote_prefix = os.environ['__remote_exec'].split() - remote_prefix.append(os.environ['__target_host']) - args[0][:0] = remote_prefix - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - raise cdist.Error("Command failed: " + " ".join(*args)) - except OSError as error: - raise cdist.Error(" ".join(*args) + ": " + error.args[1]) - - + self.remote_copy = remote_copy def remote_mkdir(self, directory): """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=True) + self.run_or_fail(["mkdir", "-p", directory], remote=True) def remove_remote_path(self, destination): """Ensure path on remote side vanished""" - cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=True) + self.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_path(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_path(destination) - cdist.exec.run_or_fail(os.environ['__remote_copy'].split() + + self.run_or_fail(os.environ['__remote_copy'].split() + ["-r", source, self.target_host + ":" + destination]) + + def shell_run_or_debug_fail(self, script, *args, remote=False, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + if remote: + remote_prefix = self.remote_exec.split() + remote_prefix.append(self.target_host) + args[0][:0] = remote_prefix + + log.debug("Shell exec cmd: %s", args) + + if 'env' in kargs: + log.debug("Shell exec env: %s", kargs['env']) + + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + + if remote: + self.run_or_fail(["cat", script], remote=remote) + + else: + try: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + except IOError as error: + raise cdist.Error(str(error)) + + raise cdist.Error("Command failed (shell): " + " ".join(*args)) + except OSError as error: + raise cdist.Error(" ".join(*args) + ": " + error.args[1]) + + def run_or_fail(self, *args, remote=False, **kargs): + if remote: + remote_prefix = self.remote_exec.split() + remote_prefix.append(self.target_host) + args[0][:0] = remote_prefix + + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + raise cdist.Error("Command failed: " + " ".join(*args)) + except OSError as error: + raise cdist.Error(" ".join(*args) + ": " + error.args[1]) + + + From a692e551c0080b8dfe71859486b37496f564c5bc Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 16:31:05 +0200 Subject: [PATCH 6/9] use new logger Signed-off-by: Steven Armstrong --- lib/cdist/exec.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index b3b86a52..13cec499 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -25,14 +25,13 @@ import subprocess import cdist -log = logging.getLogger(__name__) - class Wrapper(object): def __init__(self, target_host, remote_exec, remote_copy): self.target_host = target_host self.remote_exec = remote_exec self.remote_copy = remote_copy + self.log = logging.getLogger(self.target_host) def remote_mkdir(self, directory): """Create directory on remote side""" @@ -58,15 +57,15 @@ class Wrapper(object): remote_prefix.append(self.target_host) args[0][:0] = remote_prefix - log.debug("Shell exec cmd: %s", args) + self.log.debug("Shell exec cmd: %s", args) if 'env' in kargs: - log.debug("Shell exec env: %s", kargs['env']) + self.log.debug("Shell exec env: %s", kargs['env']) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") + self.log.error("Code that raised the error:\n") if remote: self.run_or_fail(["cat", script], remote=remote) @@ -89,13 +88,10 @@ class Wrapper(object): remote_prefix.append(self.target_host) args[0][:0] = remote_prefix - log.debug("Exec: " + " ".join(*args)) + self.log.debug("Exec: " + " ".join(*args)) try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: raise cdist.Error("Command failed: " + " ".join(*args)) except OSError as error: raise cdist.Error(" ".join(*args) + ": " + error.args[1]) - - - From 8ef0c18da0630212a50d92afcfcbcfe006b5da95 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 16:31:43 +0200 Subject: [PATCH 7/9] update test suite to match new code Signed-off-by: Steven Armstrong --- lib/cdist/test/test_exec.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/cdist/test/test_exec.py b/lib/cdist/test/test_exec.py index 901b5efd..68f388e9 100644 --- a/lib/cdist/test/test_exec.py +++ b/lib/cdist/test/test_exec.py @@ -49,32 +49,35 @@ class Exec(unittest.TestCase): false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) false_fd.close() + target_host = "does.not.exist" + remote_exec = "ssh -o User=root -q" + remote_copy = "scp -o User=root -q" + self.wrapper = cdist.exec.Wrapper(target_host, remote_exec, remote_copy) + def tearDown(self): shutil.rmtree(self.temp_dir) def test_local_success_shell(self): try: - cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + self.wrapper.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) except cdist.Error: failed = True else: failed = False - self.assertFalse(failed) def test_local_fail_shell(self): - self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, + self.assertRaises(cdist.Error, self.wrapper.shell_run_or_debug_fail, self.shell_false, [self.shell_false]) def test_local_success(self): try: - cdist.exec.run_or_fail(["/bin/true"]) + self.wrapper.run_or_fail(["/bin/true"]) except cdist.Error: failed = True else: failed = False - self.assertFalse(failed) def test_local_fail(self): - self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) + self.assertRaises(cdist.Error, self.wrapper.run_or_fail, ["/bin/false"]) From 1c73fb1288eabb1006e67626065252e6ca0bf594 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 16:41:01 +0200 Subject: [PATCH 8/9] -- unused imports Signed-off-by: Steven Armstrong --- lib/cdist/test/test_exec.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/cdist/test/test_exec.py b/lib/cdist/test/test_exec.py index 68f388e9..deb62d11 100644 --- a/lib/cdist/test/test_exec.py +++ b/lib/cdist/test/test_exec.py @@ -22,17 +22,13 @@ import os -import sys import shutil -import subprocess import tempfile import unittest -sys.path.insert(0, os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) - import cdist.exec + class Exec(unittest.TestCase): def setUp(self): """Create shell code and co.""" From 8543f3ed58fe6577b6dcc02868a607ec2bcdb833 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 11 Oct 2011 16:41:44 +0200 Subject: [PATCH 9/9] rename to match others Signed-off-by: Steven Armstrong --- lib/cdist/test/test_exec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/test/test_exec.py b/lib/cdist/test/test_exec.py index deb62d11..c2bb52f6 100644 --- a/lib/cdist/test/test_exec.py +++ b/lib/cdist/test/test_exec.py @@ -29,7 +29,7 @@ import unittest import cdist.exec -class Exec(unittest.TestCase): +class ExecTestCase(unittest.TestCase): def setUp(self): """Create shell code and co."""