From d3923c7b0f06a3cf11b9adb0a1ffc1ffa9a4cdb3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 14:15:30 +0200 Subject: [PATCH 01/42] add template for install tests Signed-off-by: Nico Schottelius --- test/test_install.py | 101 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 test/test_install.py diff --git a/test/test_install.py b/test/test_install.py new file mode 100644 index 00000000..21e8ca06 --- /dev/null +++ b/test/test_install.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) + +import cdist.config + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + + +class Install(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.config = cdist.config.Config("localhost", + initial_manifest=self.init_manifest, + exec_path=cdist_exec_path) + self.config.link_emulator() + + def test_initial_manifest_different_parameter(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_added(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + '\n', + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_removed(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + "\n", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + + def test_initial_manifest_parameter_twice(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", + ]) + manifest_fd.close() + + try: + self.config.run_initial_manifest() + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + From 2d567c175926fa40b427b3cbb9eaeaa00ff8b9da Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 14:18:39 +0200 Subject: [PATCH 02/42] prepare cleanup of tests for later Signed-off-by: Nico Schottelius --- build.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.sh b/build.sh index 021fb480..c6906357 100755 --- a/build.sh +++ b/build.sh @@ -41,6 +41,9 @@ MAN1DSTDIR=${MANDIR}/man1 MAN7DSTDIR=${MANDIR}/man7 SPEECHESDIR=doc/speeches +# FIXME: make lib for tests! +# PYTHONPATH=$PYTHONPATH:$(pwd -P)/test/lib + case "$1" in man) set -e From ffb533eae8b881ac7b67781453d60baaa1916649 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 14:18:43 +0200 Subject: [PATCH 03/42] -exec Signed-off-by: Nico Schottelius --- test/nico_ui.py | 0 test/test_exec.py | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 test/nico_ui.py mode change 100755 => 100644 test/test_exec.py diff --git a/test/nico_ui.py b/test/nico_ui.py old mode 100755 new mode 100644 diff --git a/test/test_exec.py b/test/test_exec.py old mode 100755 new mode 100644 From db322c0b5e52849c99226bec383978f3fbab2a65 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 14:29:33 +0200 Subject: [PATCH 04/42] integrate tests into library Signed-off-by: Nico Schottelius --- lib/cdist/test/__init__.py | 0 {test => lib/cdist/test}/nico_ui.py | 0 {test => lib/cdist/test}/test_config.py | 0 {test => lib/cdist/test}/test_exec.py | 0 {test => lib/cdist/test}/test_install.py | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/cdist/test/__init__.py rename {test => lib/cdist/test}/nico_ui.py (100%) rename {test => lib/cdist/test}/test_config.py (100%) rename {test => lib/cdist/test}/test_exec.py (100%) rename {test => lib/cdist/test}/test_install.py (100%) diff --git a/lib/cdist/test/__init__.py b/lib/cdist/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/test/nico_ui.py b/lib/cdist/test/nico_ui.py similarity index 100% rename from test/nico_ui.py rename to lib/cdist/test/nico_ui.py diff --git a/test/test_config.py b/lib/cdist/test/test_config.py similarity index 100% rename from test/test_config.py rename to lib/cdist/test/test_config.py diff --git a/test/test_exec.py b/lib/cdist/test/test_exec.py similarity index 100% rename from test/test_exec.py rename to lib/cdist/test/test_exec.py diff --git a/test/test_install.py b/lib/cdist/test/test_install.py similarity index 100% rename from test/test_install.py rename to lib/cdist/test/test_install.py From f5a58f768371be278dd66760c87b43127c5acb6f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 14:30:38 +0200 Subject: [PATCH 05/42] adjust auto discovery Signed-off-by: Nico Schottelius --- build.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index c6906357..39192c78 100755 --- a/build.sh +++ b/build.sh @@ -130,11 +130,11 @@ case "$1" in ;; test) - python3 -m unittest discover test 'test_*.py' + python3 -m unittest discover lib/cdist/test 'test_*.py' ;; test-all) - python3 -m unittest discover test '*.py' + python3 -m unittest discover lib/cdist/test '*.py' ;; *) From 13ed37a4e954d60e27199bfc981d7b795fb36302 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 14:45:35 +0200 Subject: [PATCH 06/42] begin to test explorer success in test_install Signed-off-by: Nico Schottelius --- build.sh | 8 +++++--- lib/cdist/test/test_install.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/build.sh b/build.sh index 39192c78..01094e12 100755 --- a/build.sh +++ b/build.sh @@ -41,9 +41,6 @@ MAN1DSTDIR=${MANDIR}/man1 MAN7DSTDIR=${MANDIR}/man7 SPEECHESDIR=doc/speeches -# FIXME: make lib for tests! -# PYTHONPATH=$PYTHONPATH:$(pwd -P)/test/lib - case "$1" in man) set -e @@ -133,6 +130,11 @@ case "$1" in python3 -m unittest discover lib/cdist/test 'test_*.py' ;; + test-install) + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ + python3 -m unittest cdist.test.test_install + ;; + test-all) python3 -m unittest discover lib/cdist/test '*.py' ;; diff --git a/lib/cdist/test/test_install.py b/lib/cdist/test/test_install.py index 21e8ca06..f5956585 100644 --- a/lib/cdist/test/test_install.py +++ b/lib/cdist/test/test_install.py @@ -43,6 +43,18 @@ class Install(unittest.TestCase): exec_path=cdist_exec_path) self.config.link_emulator() +### NEW FOR INSTALL ############################################################ + + def test_explorer_ran(self): + """Check that all explorers returned a result""" + self.config.run_global_explores() + explorers = self.config.path.list_global_explorers() + + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + self.assertTrue(os.path.isfile(output)) + +### OLD FROM CONFIG ############################################################ def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", From 410a2fe7ff1f56dc54420c921fff6390af357e62 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 15:08:37 +0200 Subject: [PATCH 07/42] test_explorer_ran finished Signed-off-by: Nico Schottelius --- lib/cdist/test/test_install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/test/test_install.py b/lib/cdist/test/test_install.py index f5956585..3289ea47 100644 --- a/lib/cdist/test/test_install.py +++ b/lib/cdist/test/test_install.py @@ -51,7 +51,7 @@ class Install(unittest.TestCase): explorers = self.config.path.list_global_explorers() for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) + output = self.config.path.global_explorer_output_path(explorer) self.assertTrue(os.path.isfile(output)) ### OLD FROM CONFIG ############################################################ From ec82fa8f972b13cec907d8fff4560ff2fa3201ad Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 15:09:33 +0200 Subject: [PATCH 08/42] document tests/single ones Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-04 | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/dev/logs/2011-10-04 diff --git a/doc/dev/logs/2011-10-04 b/doc/dev/logs/2011-10-04 new file mode 100644 index 00000000..f3bb852d --- /dev/null +++ b/doc/dev/logs/2011-10-04 @@ -0,0 +1,3 @@ +Testing for single tests: + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib python3 -m unittest cdist.test.test_install.Install.test_explorer_ran + From 3d75ec9bfcde3b3f057d1d7c4061c1e553c3e681 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 15:49:18 +0200 Subject: [PATCH 09/42] make test suite usable from command line Signed-off-by: Nico Schottelius --- lib/cdist/test/__main__.py | 41 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 lib/cdist/test/__main__.py diff --git a/lib/cdist/test/__main__.py b/lib/cdist/test/__main__.py new file mode 100644 index 00000000..03bca847 --- /dev/null +++ b/lib/cdist/test/__main__.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import sys +import cdist.test + +#class UI(unittest.TestCase): +# def test_banner(self): +# self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) +# +# def test_help(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) +# +# # FIXME: mockup needed +# def test_config_localhost(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + +print(cdist.test.cdist_exec_path) +print(sys.argv) From 40a1619c1ad6731666a6c3303e4550958e43ef45 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 16:15:10 +0200 Subject: [PATCH 10/42] make build test work (or fail, but work) again Signed-off-by: Nico Schottelius --- build.sh | 6 +++-- lib/cdist/test/__init__.py | 47 ++++++++++++++++++++++++++++++++++++++ lib/cdist/test/__main__.py | 5 ++++ 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/build.sh b/build.sh index 01094e12..6a91ff3d 100755 --- a/build.sh +++ b/build.sh @@ -127,7 +127,8 @@ case "$1" in ;; test) - python3 -m unittest discover lib/cdist/test 'test_*.py' + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ + python3 -m cdist.test ;; test-install) @@ -136,7 +137,8 @@ case "$1" in ;; test-all) - python3 -m unittest discover lib/cdist/test '*.py' + PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib \ + python3 -m unittest discover lib/cdist/test '*.py' ;; *) diff --git a/lib/cdist/test/__init__.py b/lib/cdist/test/__init__.py index e69de29b..f614fa05 100644 --- a/lib/cdist/test/__init__.py +++ b/lib/cdist/test/__init__.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import subprocess +import unittest + +cdist_commands=["banner", "config", "install"] + +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../bin/cdist")) + +def exec(): + print(cdist_exec_path) + +#class UI(unittest.TestCase): +# def test_banner(self): +# self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) +# +# def test_help(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) +# +# # FIXME: mockup needed +# def test_config_localhost(self): +# for cmd in cdist_commands: +# self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) diff --git a/lib/cdist/test/__main__.py b/lib/cdist/test/__main__.py index 03bca847..3b31a2cd 100644 --- a/lib/cdist/test/__main__.py +++ b/lib/cdist/test/__main__.py @@ -21,8 +21,10 @@ # +import os import sys import cdist.test +import unittest #class UI(unittest.TestCase): # def test_banner(self): @@ -39,3 +41,6 @@ import cdist.test print(cdist.test.cdist_exec_path) print(sys.argv) + +suite = unittest.defaultTestLoader.discover(os.path.dirname(__file__)) +unittest.TextTestRunner(verbosity=1).run(suite) From 6c22867fc253096f8134960fbb0b4c68f05dd44b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 16:32:43 +0200 Subject: [PATCH 11/42] begin to test path and add method to check type Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 6 +++ lib/cdist/test/test_install.py | 16 ++++++++ lib/cdist/test/test_path.py | 71 ++++++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+) create mode 100644 lib/cdist/test/test_path.py diff --git a/lib/cdist/path.py b/lib/cdist/path.py index e416c42d..2dd9dcf1 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -175,8 +175,14 @@ class Path: return list def list_types(self): + """Retuns list of types""" return os.listdir(self.type_base_dir) + def is_install_type(self, type): + """Check whether a type is used for installation (if not: for configuration)""" + marker = os.path.join(self.type_dir(type), "install") + return os.path.isfile(marker) + def list_object_paths(self, starting_point): """Return list of paths of existing objects""" object_paths = [] diff --git a/lib/cdist/test/test_install.py b/lib/cdist/test/test_install.py index 3289ea47..9cfae066 100644 --- a/lib/cdist/test/test_install.py +++ b/lib/cdist/test/test_install.py @@ -54,6 +54,22 @@ class Install(unittest.TestCase): output = self.config.path.global_explorer_output_path(explorer) self.assertTrue(os.path.isfile(output)) + def test_manifest_uses_install_types_only(self): + """Check that objects created from manifest are only of install type""" + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__partition_msdos /dev/null --type 82\n", + ]) + manifest_fd.close() + + self.config.run_initial_manifest() + + # FIXME: check that only __partition_msdos objects are created! + + self.assertFalse(failed) + + ### OLD FROM CONFIG ############################################################ def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") diff --git a/lib/cdist/test/test_path.py b/lib/cdist/test/test_path.py new file mode 100644 index 00000000..04a107b8 --- /dev/null +++ b/lib/cdist/test/test_path.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import sys +import tempfile +import unittest + +import cdist.path +import cdist.test + +class Path(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.path = cdist.config.Path("localhost", "root", + "ssh root@localhost", + initial_manifest=self.init_manifest, + base_dir=self.temp_dir) + + def tearDown(self): + self.path.cleanup() + + def test_type_detection(self): + """Check that a type is identified as install or configuration correctly""" + + # Create install type + install_type = os.path.join( + os.mkdir( + # Create non-install type + + self.config.run_global_explores() + explorers = self.config.path.list_global_explorers() + + for explorer in explorers: + output = self.config.path.global_explorer_output_path(explorer) + self.assertTrue(os.path.isfile(output)) + + def test_manifest_uses_install_types_only(self): + """Check that objects created from manifest are only of install type""" + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__partition_msdos /dev/null --type 82\n", + ]) + manifest_fd.close() + + self.config.run_initial_manifest() + + # FIXME: check that only __partition_msdos objects are created! + + self.assertFalse(failed) From 8dd248cf7664c2cc428c7639b26b102a20e6cbea Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 17:01:11 +0200 Subject: [PATCH 12/42] cleanup on exit and begin to create test types Signed-off-by: Nico Schottelius --- lib/cdist/test/test_path.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/cdist/test/test_path.py b/lib/cdist/test/test_path.py index 04a107b8..1fd955e3 100644 --- a/lib/cdist/test/test_path.py +++ b/lib/cdist/test/test_path.py @@ -21,6 +21,7 @@ # import os +import shutil import sys import tempfile import unittest @@ -32,28 +33,29 @@ class Path(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() self.init_manifest = os.path.join(self.temp_dir, "manifest") - self.path = cdist.config.Path("localhost", "root", - "ssh root@localhost", + self.path = cdist.path.Path("localhost", "root", "ssh root@localhost", initial_manifest=self.init_manifest, base_dir=self.temp_dir) + os.mkdir(self.path.conf_dir) + os.mkdir(self.path.type_base_dir) + + # Create install type + self.install_type = os.path.join(self.path.type_base_dir, "__install_test") + os.mkdir(self.install_type) + open(os.path.join(self.install_type, "install"), "w").close() + + # Create config type + config_type = os.path.join(self.path.type_base_dir, "__config_test") + os.mkdir(config_type) + def tearDown(self): self.path.cleanup() + shutil.rmtree(self.temp_dir) def test_type_detection(self): """Check that a type is identified as install or configuration correctly""" - # Create install type - install_type = os.path.join( - os.mkdir( - # Create non-install type - - self.config.run_global_explores() - explorers = self.config.path.list_global_explorers() - - for explorer in explorers: - output = self.config.path.global_explorer_output_path(explorer) - self.assertTrue(os.path.isfile(output)) def test_manifest_uses_install_types_only(self): """Check that objects created from manifest are only of install type""" From f88cb78d61bd803b183aab450075e0b1bdee8304 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 18:36:43 +0200 Subject: [PATCH 13/42] finish test_path.test_type_detection Signed-off-by: Nico Schottelius --- lib/cdist/test/test_path.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/cdist/test/test_path.py b/lib/cdist/test/test_path.py index 1fd955e3..c8ca95c0 100644 --- a/lib/cdist/test/test_path.py +++ b/lib/cdist/test/test_path.py @@ -40,14 +40,17 @@ class Path(unittest.TestCase): os.mkdir(self.path.conf_dir) os.mkdir(self.path.type_base_dir) + self.install_type_name = "__install_test" + self.config_type_name = "__config_test" + # Create install type - self.install_type = os.path.join(self.path.type_base_dir, "__install_test") + self.install_type = os.path.join(self.path.type_base_dir, self.install_type_name) os.mkdir(self.install_type) open(os.path.join(self.install_type, "install"), "w").close() # Create config type - config_type = os.path.join(self.path.type_base_dir, "__config_test") - os.mkdir(config_type) + self.config_type = os.path.join(self.path.type_base_dir, self.config_type_name) + os.mkdir(self.config_type) def tearDown(self): self.path.cleanup() @@ -55,7 +58,9 @@ class Path(unittest.TestCase): def test_type_detection(self): """Check that a type is identified as install or configuration correctly""" - + + self.assertTrue(self.path.is_install_type(self.install_type)) + self.assertFalse(self.path.is_install_type(self.config_type)) def test_manifest_uses_install_types_only(self): """Check that objects created from manifest are only of install type""" From b3f914f6f6fa2ef0ce390622d5b5cb4438000270 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Oct 2011 18:45:29 +0200 Subject: [PATCH 14/42] create base module for install and config Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 268 +-------------------------------- lib/cdist/config_install.py | 292 ++++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+), 265 deletions(-) create mode 100644 lib/cdist/config_install.py diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 51615c28..fcc9ed7e 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -22,274 +22,12 @@ import datetime import logging -import os -import stat -import sys - log = logging.getLogger(__name__) -import cdist.emulator -import cdist.path +import cdist.config_install -CODE_HEADER = "#!/bin/sh -e\n" - -class Config: - """Cdist main class to hold arbitrary data""" - - def __init__(self, target_host, - initial_manifest=False, - remote_user="root", - home=None, - exec_path=sys.argv[0], - debug=False): - - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user - self.exec_path = exec_path - - # FIXME: broken - construct elsewhere! - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] - - self.path = cdist.path.Path(self.target_host, - initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, - base_dir=home, - debug=debug) - - self.objects_prepared = [] - - def cleanup(self): - self.path.cleanup() - - def run_global_explores(self): - """Run global explorers""" - log.info("Running global explorers") - explorers = self.path.list_global_explorers() - if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) - - self.path.transfer_global_explorers() - for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.path.remote_global_explorer_path(explorer)) - - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - - type = self.path.get_type_from_object(cdist_object) - self.path.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.path.transfer_object_parameter(cdist_object) - - explorers = self.path.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() - - def link_emulator(self): - """Link emulator to types""" - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - self.link_emulator() - - def run_initial_manifest(self): - """Run the initial manifest""" - log.info("Running initial manifest %s", self.path.initial_manifest) - env = { "__manifest" : self.path.manifest_dir } - self.run_manifest(self.path.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.path.get_type_from_object(cdist_object) - manifest = self.path.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.path.object_dir(cdist_object), - "__object_id": self.path.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.path.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.path.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - - # Submit debug flag to manifest, can be used by emulator and types - if self.debug: - env['__debug'] = "yes" - - # Required for recording source - env['__cdist_manifest'] = manifest - - # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir - - # Other environment stuff - if extra_env: - env.update(extra_env) - - cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) - - if mode == "gencode": - paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - log.info("Running object manifests and type explorers") - - old_objects = [] - objects = self.path.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = self.path.list_objects() - - def stage_run(self): - """The final (and real) step of deployment""" - log.info("Generating and executing code") - # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - - self.stage_prepare() - self.stage_run() - - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() +class Config(cdist.config_install.ConfigInstall): + pass def config(args): """Configure remote system""" diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py new file mode 100644 index 00000000..f7bd43e8 --- /dev/null +++ b/lib/cdist/config_install.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import datetime +import logging +import os +import stat +import sys + +log = logging.getLogger(__name__) + +import cdist.emulator +import cdist.path + +CODE_HEADER = "#!/bin/sh -e\n" + +class ConfigInstall: + """Class to hold install and config methods""" + + def __init__(self, target_host, + initial_manifest=False, + remote_user="root", + home=None, + exec_path=sys.argv[0], + debug=False): + + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.exec_path = exec_path + + # FIXME: broken - construct elsewhere! + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] + + def cleanup(self): + self.path.cleanup() + + def run_global_explores(self): + """Run global explorers""" + log.info("Running global explorers") + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) + + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) + + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) + + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() + + def run_initial_manifest(self): + """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" + + # Required for recording source + env['__cdist_manifest'] = manifest + + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) + + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") + + old_objects = [] + objects = self.path.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.path.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() From acc64caf95524156686f3aabbe7fea1b9c4fab60 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 22:23:26 +0200 Subject: [PATCH 15/42] new type: __mkfs Signed-off-by: Steven Armstrong --- conf/type/__mkfs/gencode-remote | 36 ++++++++++++++++++ conf/type/__mkfs/installer | 0 conf/type/__mkfs/man.text | 57 +++++++++++++++++++++++++++++ conf/type/__mkfs/manifest | 31 ++++++++++++++++ conf/type/__mkfs/parameter/optional | 3 ++ conf/type/__mkfs/parameter/required | 1 + 6 files changed, 128 insertions(+) create mode 100755 conf/type/__mkfs/gencode-remote create mode 100644 conf/type/__mkfs/installer create mode 100644 conf/type/__mkfs/man.text create mode 100755 conf/type/__mkfs/manifest create mode 100644 conf/type/__mkfs/parameter/optional create mode 100644 conf/type/__mkfs/parameter/required diff --git a/conf/type/__mkfs/gencode-remote b/conf/type/__mkfs/gencode-remote new file mode 100755 index 00000000..e5061013 --- /dev/null +++ b/conf/type/__mkfs/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +device="(cat "$__object/parameter/device")" +type="(cat "$__object/parameter/type")" + +command="mkfs -t $type" + +if [ -f "$__object/parameter/options" ]; then + options="(cat "$__object/parameter/options")" + command="$command -o '$options'" +fi +command="$command $device" +if [ -f "$__object/parameter/blocks" ]; then + blocks="(cat "$__object/parameter/blocks")" + command="$command $blocks" +fi + +echo "$command" diff --git a/conf/type/__mkfs/installer b/conf/type/__mkfs/installer new file mode 100644 index 00000000..e69de29b diff --git a/conf/type/__mkfs/man.text b/conf/type/__mkfs/man.text new file mode 100644 index 00000000..4320c639 --- /dev/null +++ b/conf/type/__mkfs/man.text @@ -0,0 +1,57 @@ +cdist-type__mkfs(7) +=================== +Steven Armstrong + + +NAME +---- +cdist-type__mkfs - build a linux file system + + +DESCRIPTION +----------- +This cdist type is a wrapper for the mkfs command. + + +REQUIRED PARAMETERS +------------------- +type:: + The filesystem type to use. Same as mkfs -t. + + +OPTIONAL PARAMETERS +------------------- +device:: + defaults to object_id + +options:: + file system-specific options to be passed to the mkfs command + +blocks:: + the number of blocks to be used for the file system + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# reiserfs /dev/sda5 +__mkfs /dev/sda5 --type reiserfs +# same thing with explicit device +__mkfs whatever --device /dev/sda5 --type reiserfs + +# jfs with journal on /dev/sda2 +__mkfs /dev/sda1 --type jfs --options "-j /dev/sda2" +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- mkfs(8) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/conf/type/__mkfs/manifest b/conf/type/__mkfs/manifest new file mode 100755 index 00000000..e9d275a4 --- /dev/null +++ b/conf/type/__mkfs/manifest @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +if [ -f "$__object/parameter/device" ]; then + device="(cat "$__object/parameter/device")" +else + device="/$__object_id" + echo "$device" > "$__object/parameter/device" +fi + +type="(cat "$__object/parameter/type")" + +options="(cat "$__object/parameter/options")" diff --git a/conf/type/__mkfs/parameter/optional b/conf/type/__mkfs/parameter/optional new file mode 100644 index 00000000..86aeae30 --- /dev/null +++ b/conf/type/__mkfs/parameter/optional @@ -0,0 +1,3 @@ +device +options +blocks diff --git a/conf/type/__mkfs/parameter/required b/conf/type/__mkfs/parameter/required new file mode 100644 index 00000000..aa80e646 --- /dev/null +++ b/conf/type/__mkfs/parameter/required @@ -0,0 +1 @@ +type From 27a774432e900896fc2a9a5e73adec794e13c296 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 22:42:19 +0200 Subject: [PATCH 16/42] mark __partition_msdos and __partition_msdos_apply as installer types Signed-off-by: Steven Armstrong --- conf/type/__partition_msdos/installer | 0 conf/type/__partition_msdos_apply/installer | 0 2 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 conf/type/__partition_msdos/installer create mode 100644 conf/type/__partition_msdos_apply/installer diff --git a/conf/type/__partition_msdos/installer b/conf/type/__partition_msdos/installer new file mode 100644 index 00000000..e69de29b diff --git a/conf/type/__partition_msdos_apply/installer b/conf/type/__partition_msdos_apply/installer new file mode 100644 index 00000000..e69de29b From 9fd74acfac2194874224ea046953094cd56dfffa Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 22:50:49 +0200 Subject: [PATCH 17/42] add support for swap Signed-off-by: Steven Armstrong --- conf/type/__mkfs/gencode-remote | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/conf/type/__mkfs/gencode-remote b/conf/type/__mkfs/gencode-remote index e5061013..0a220ec9 100755 --- a/conf/type/__mkfs/gencode-remote +++ b/conf/type/__mkfs/gencode-remote @@ -21,16 +21,18 @@ device="(cat "$__object/parameter/device")" type="(cat "$__object/parameter/type")" -command="mkfs -t $type" - -if [ -f "$__object/parameter/options" ]; then - options="(cat "$__object/parameter/options")" - command="$command -o '$options'" +if [ "$type" = "swap" ]; then + echo "mkswap $device" +else + command="mkfs -t $type" + if [ -f "$__object/parameter/options" ]; then + options="(cat "$__object/parameter/options")" + command="$command -o '$options'" + fi + command="$command $device" + if [ -f "$__object/parameter/blocks" ]; then + blocks="(cat "$__object/parameter/blocks")" + command="$command $blocks" + fi + echo "$command" fi -command="$command $device" -if [ -f "$__object/parameter/blocks" ]; then - blocks="(cat "$__object/parameter/blocks")" - command="$command $blocks" -fi - -echo "$command" From 20241a0c5c4d005f66ac2591e8bddcebb9ba8ab6 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 22:56:24 +0200 Subject: [PATCH 18/42] fix copy/paste error Signed-off-by: Steven Armstrong --- conf/type/__mkfs/gencode-remote | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conf/type/__mkfs/gencode-remote b/conf/type/__mkfs/gencode-remote index 0a220ec9..67e37cce 100755 --- a/conf/type/__mkfs/gencode-remote +++ b/conf/type/__mkfs/gencode-remote @@ -18,20 +18,20 @@ # along with cdist. If not, see . # -device="(cat "$__object/parameter/device")" -type="(cat "$__object/parameter/type")" +device="$(cat "$__object/parameter/device")" +type="$(cat "$__object/parameter/type")" if [ "$type" = "swap" ]; then echo "mkswap $device" else command="mkfs -t $type" if [ -f "$__object/parameter/options" ]; then - options="(cat "$__object/parameter/options")" + options="$(cat "$__object/parameter/options")" command="$command -o '$options'" fi command="$command $device" if [ -f "$__object/parameter/blocks" ]; then - blocks="(cat "$__object/parameter/blocks")" + blocks="$(cat "$__object/parameter/blocks")" command="$command $blocks" fi echo "$command" From 3d017abd6f207a819b9fc90fc043bbe1be91accc Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 22:56:50 +0200 Subject: [PATCH 19/42] pass options to mkfs without -o Signed-off-by: Steven Armstrong --- conf/type/__mkfs/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__mkfs/gencode-remote b/conf/type/__mkfs/gencode-remote index 67e37cce..e774f33d 100755 --- a/conf/type/__mkfs/gencode-remote +++ b/conf/type/__mkfs/gencode-remote @@ -27,7 +27,7 @@ else command="mkfs -t $type" if [ -f "$__object/parameter/options" ]; then options="$(cat "$__object/parameter/options")" - command="$command -o '$options'" + command="$command $options" fi command="$command $device" if [ -f "$__object/parameter/blocks" ]; then From de4ddf9d1e4cdce8376e1bc87381c70de8cf6eba Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 23:03:50 +0200 Subject: [PATCH 20/42] mv installer install Signed-off-by: Steven Armstrong --- conf/type/__mkfs/gencode-remote | 2 +- conf/type/__mkfs/{installer => install} | 0 conf/type/__partition_msdos/{installer => install} | 0 conf/type/__partition_msdos_apply/{installer => install} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename conf/type/__mkfs/{installer => install} (100%) rename conf/type/__partition_msdos/{installer => install} (100%) rename conf/type/__partition_msdos_apply/{installer => install} (100%) diff --git a/conf/type/__mkfs/gencode-remote b/conf/type/__mkfs/gencode-remote index e774f33d..b3561bad 100755 --- a/conf/type/__mkfs/gencode-remote +++ b/conf/type/__mkfs/gencode-remote @@ -24,7 +24,7 @@ type="$(cat "$__object/parameter/type")" if [ "$type" = "swap" ]; then echo "mkswap $device" else - command="mkfs -t $type" + command="mkfs -t $type -q" if [ -f "$__object/parameter/options" ]; then options="$(cat "$__object/parameter/options")" command="$command $options" diff --git a/conf/type/__mkfs/installer b/conf/type/__mkfs/install similarity index 100% rename from conf/type/__mkfs/installer rename to conf/type/__mkfs/install diff --git a/conf/type/__partition_msdos/installer b/conf/type/__partition_msdos/install similarity index 100% rename from conf/type/__partition_msdos/installer rename to conf/type/__partition_msdos/install diff --git a/conf/type/__partition_msdos_apply/installer b/conf/type/__partition_msdos_apply/install similarity index 100% rename from conf/type/__partition_msdos_apply/installer rename to conf/type/__partition_msdos_apply/install From f0223647e34c642aa24abb5b08db0cee4ae05154 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 23:12:49 +0200 Subject: [PATCH 21/42] --debug Signed-off-by: Steven Armstrong --- conf/type/__partition_msdos_apply/files/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__partition_msdos_apply/files/lib.sh b/conf/type/__partition_msdos_apply/files/lib.sh index f0859aab..021c11d4 100644 --- a/conf/type/__partition_msdos_apply/files/lib.sh +++ b/conf/type/__partition_msdos_apply/files/lib.sh @@ -3,7 +3,7 @@ die() { exit 1 } debug() { - echo "[__partition_msdos_apply] $@" >&2 + #echo "[__partition_msdos_apply] $@" >&2 } fdisk_command() { From 9d6a00af3885f8e8f23f450c0b589c2660c1ef12 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 23:13:53 +0200 Subject: [PATCH 22/42] --debug without creating syntax errors Signed-off-by: Steven Armstrong --- conf/type/__partition_msdos_apply/files/lib.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/conf/type/__partition_msdos_apply/files/lib.sh b/conf/type/__partition_msdos_apply/files/lib.sh index 021c11d4..d7f07060 100644 --- a/conf/type/__partition_msdos_apply/files/lib.sh +++ b/conf/type/__partition_msdos_apply/files/lib.sh @@ -4,6 +4,7 @@ die() { } debug() { #echo "[__partition_msdos_apply] $@" >&2 + : } fdisk_command() { From 648f57173c7fa1f405e62bd7f441c88f2e83b758 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 4 Oct 2011 23:53:06 +0200 Subject: [PATCH 23/42] bugfix: use -gt when comparing int values Signed-off-by: Steven Armstrong --- conf/type/__partition_msdos_apply/files/lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__partition_msdos_apply/files/lib.sh b/conf/type/__partition_msdos_apply/files/lib.sh index d7f07060..5767ea43 100644 --- a/conf/type/__partition_msdos_apply/files/lib.sh +++ b/conf/type/__partition_msdos_apply/files/lib.sh @@ -52,7 +52,7 @@ create_partition() { first_minor="${minor}\n" type_minor="${minor}\n" primary_extended="l\n" - [ "$primary_count" > "3" ] && primary_extended="" + [ "$primary_count" -gt "3" ] && primary_extended="" fi [ -n "${size}" ] && size="+${size}M" fdisk_command ${device} "n\n${primary_extended}${first_minor}\n${size}\nt\n${type_minor}${type}\n" From 8a044919eb9e74dde6651ccd60ce390004ab8e97 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 13:24:07 +0200 Subject: [PATCH 24/42] +discussion about restructering/ "object-orientation" Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-05 | 112 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 doc/dev/logs/2011-10-05 diff --git a/doc/dev/logs/2011-10-05 b/doc/dev/logs/2011-10-05 new file mode 100644 index 00000000..92e8675f --- /dev/null +++ b/doc/dev/logs/2011-10-05 @@ -0,0 +1,112 @@ +Config/Install/Deploy/Run: + target host + remote_cmd_prefix - ssh user@bla sudo foo????? + remote_cp_prefix - cp statt scp oder so + + debug -> env für alles += __debug + + +Storage/Metaobject/Tree? == Path? + base_dir? + nimmt objekte + + Sammelt Objekte + + Ist prepared hier? + +Object + "Infos" / Datenhalde + + Base_Dir-Abhängigkeit? - wo + + out_dir - wo speichern + + nur eigenes verzeichnis interessant? + -> nicht für shell code / aka gencode! + -> __global abhängigkeit + + object.gencode()? + + hast du type-explorer? + ja? + führe JEDEN remote aus + speichere ausgabe in object + nein: + fertig + hast du gencode-{local,remote}? + ja? + führe local oder remote aus + speichere ausgabe in s/^gen// + nein: + fertig + + hast du code-{local,remote}? + ja? + führe local oder remote aus + nein: + fertig + + ich habe ... + object_id + type + type.singleton() == False -> require object_id + parameter gegeben + requirements / order + + type_explorer := methode zum ausführen? + + cdist.object.Object(type, id) + + methoden: + gen_code + code + run_manifest + manifest == ort + +Type + singleton: ja / nein + install: ja / nein + type_explorer := liste + + optional_parameter + required_parameter + + TypeExplorer + verwandt oder == explorer + Verwandschaft klären! + + sehr abhängig von base_dir! + - welche gibt es? + - was für optionen haben sie + + cdist.type.Type("/path/to/type") + Tree/Path vieh, das liste von $_ speichert + Einfach iterieren + + + +Explorer + execute(env) + env == __explorer -> nur im explorer + +z.B. BaseExplorer oder andersherum GlobalExplorer + +Manifest + +Exec + wrapper um auszuführen, + error handling, + output redirection (variable, file, beides, socat :-) + + +-------------------------------------------------------------------------------- + +- base_dir (conf/, type, ...) +- manifest (initiale) + $methode_mit_inhalt_von_manifest? + run_manifest(code) + ob sinnvoll? + geht auch mit stdin oder datei + + stdin -> muss in tmp-datei, für sh -e? +- From f00b8fe56a8cb45116b9595c212c9a54796b2007 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 13:31:57 +0200 Subject: [PATCH 25/42] buffering/output idea Signed-off-by: Nico Schottelius --- doc/dev/logs/2011-10-05 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/dev/logs/2011-10-05 b/doc/dev/logs/2011-10-05 index 92e8675f..39fc48a2 100644 --- a/doc/dev/logs/2011-10-05 +++ b/doc/dev/logs/2011-10-05 @@ -110,3 +110,7 @@ Exec stdin -> muss in tmp-datei, für sh -e? - +-------------------------------------------------------------------------------- + +save output of shell in buffer instead of displaying? + -> freedom to decide whether to display or not! From a875d69d7e9dc6dabdcd33c0ee4d33dd13640376 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 14:15:05 +0200 Subject: [PATCH 26/42] continue on test_path testcase Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 5 +++-- lib/cdist/install.py | 6 ++++++ lib/cdist/test/test_path.py | 6 +++--- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index f7bd43e8..cd8ea720 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -26,11 +26,11 @@ import os import stat import sys -log = logging.getLogger(__name__) - import cdist.emulator import cdist.path +log = logging.getLogger(__name__) + CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: @@ -203,6 +203,7 @@ class ConfigInstall: outfile_fd = open(outfile, "w") # Need to flush to ensure our write is done before stdout write + # FIXME: CODE_HEADER needed in our sh -e scenario???? outfile_fd.write(CODE_HEADER) outfile_fd.flush() diff --git a/lib/cdist/install.py b/lib/cdist/install.py index 98b388ec..5a35626d 100644 --- a/lib/cdist/install.py +++ b/lib/cdist/install.py @@ -22,8 +22,14 @@ import logging +import cdist.config_install + log = logging.getLogger(__name__) + +Class Install(cdist.config_install.ConfigInstall): + pass + def install(args): """Install remote system""" process = {} diff --git a/lib/cdist/test/test_path.py b/lib/cdist/test/test_path.py index c8ca95c0..f86c8fad 100644 --- a/lib/cdist/test/test_path.py +++ b/lib/cdist/test/test_path.py @@ -66,12 +66,12 @@ class Path(unittest.TestCase): """Check that objects created from manifest are only of install type""" manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + " --mode 0700\n", - "__partition_msdos /dev/null --type 82\n", + self.install_type_name + "testid\n", + self.config_type_name + "testid\n", ]) manifest_fd.close() - self.config.run_initial_manifest() + self.install.run_initial_manifest() # FIXME: check that only __partition_msdos objects are created! From 9128cc28c414e1fa1abe4a482f6777f753123910 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 15:50:17 +0200 Subject: [PATCH 27/42] introduce a lot todo in path Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 2dd9dcf1..5cde357b 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -107,6 +107,7 @@ class Path: else: self.initial_manifest = os.path.join(self.manifest_dir, "init") + # FIXME: stays def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -120,13 +121,16 @@ class Path: shutil.move(self.temp_dir, self.cache_dir) + # FIXME: belongs to here - clearify remote* def remote_mkdir(self, directory): """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) + # FIXME: belongs to here - clearify remote* def remove_remote_dir(self, destination): cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) + # FIXME: belongs to here - clearify remote* def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) @@ -135,6 +139,7 @@ class Path: self.target_host + ":" + destination]) + # FIXME: belongs to here - clearify remote* def transfer_file(self, source, destination): """Transfer file""" cdist.exec.run_or_fail(["scp", "-q", source, @@ -142,10 +147,12 @@ class Path: self.target_host + ":" + destination]) + # FIXME: Explorer or stays def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) + # FIXME: object def type_explorer_output_dir(self, cdist_object): """Returns and creates dir of the output for a type explorer""" dir = os.path.join(self.object_dir(cdist_object), "explorer") @@ -154,14 +161,17 @@ class Path: return dir + # FIXME Stays here / Explorer? def remote_global_explorer_path(self, explorer): """Returns path to the remote explorer""" return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + # FIXME: stays here def list_global_explorers(self): """Return list of available explorers""" return os.listdir(self.global_explorer_dir) + # FIXME: Type - only needs to know its path def list_type_explorers(self, type): """Return list of available explorers for a specific type""" dir = self.type_dir(type, "explorer") @@ -174,15 +184,18 @@ class Path: return list + # Stays here def list_types(self): """Retuns list of types""" return os.listdir(self.type_base_dir) + # FIXME: type def is_install_type(self, type): """Check whether a type is used for installation (if not: for configuration)""" marker = os.path.join(self.type_dir(type), "install") return os.path.isfile(marker) + # Stays here def list_object_paths(self, starting_point): """Return list of paths of existing objects""" object_paths = [] @@ -198,36 +211,43 @@ class Path: return object_paths - # FIXME + # FIXME: Object def get_type_from_object(self, cdist_object): """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] + # FIXME: Object def get_object_id_from_object(self, cdist_object): """Returns everything but the first part (i.e. object_id) of an object""" return os.sep.join(cdist_object.split(os.sep)[1:]) + # FIXME: Object def object_dir(self, cdist_object): """Returns the full path to the object (including .cdist)""" return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + # FIXME: Object def remote_object_dir(self, cdist_object): """Returns the remote full path to the object (including .cdist)""" return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + # FIXME: Object def object_parameter_dir(self, cdist_object): """Returns the dir to the object parameter""" return os.path.join(self.object_dir(cdist_object), "parameter") + # FIXME: object def remote_object_parameter_dir(self, cdist_object): """Returns the remote dir to the object parameter""" return os.path.join(self.remote_object_dir(cdist_object), "parameter") + # FIXME: object def object_code_paths(self, cdist_object): """Return paths to code scripts of object""" return [os.path.join(self.object_dir(cdist_object), "code-local"), os.path.join(self.object_dir(cdist_object), "code-remote")] + # Stays here def list_objects(self): """Return list of existing objects""" @@ -240,14 +260,17 @@ class Path: return objects + # FIXME: Type def type_dir(self, type, *args): - """Return directory the type""" + """Return (sub-)directory of a type""" return os.path.join(self.type_base_dir, type, *args) + # FIXME: Type def remote_type_explorer_dir(self, type): """Return remote directory that holds the explorers of a type""" return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + # Stays here def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" # Create base path before using mkdir -p @@ -257,11 +280,13 @@ class Path: self.transfer_dir(self.object_parameter_dir(cdist_object), self.remote_object_parameter_dir(cdist_object)) + # Stays here def transfer_global_explorers(self): """Transfer the global explorers""" self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + # Stays here def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" if type in self.type_explorers_transferred: From 28428177ae8734c3b4eb84d361f2c6447c58debc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 15:53:34 +0200 Subject: [PATCH 28/42] --typo Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 426 ++++++++++++++++++------------------ 1 file changed, 213 insertions(+), 213 deletions(-) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index cd8ea720..a550c45e 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -31,263 +31,263 @@ import cdist.path log = logging.getLogger(__name__) -CODE_HEADER = "#!/bin/sh -e\n" +CODE_HEADER = "#!/bin/sh -e\n" class ConfigInstall: - """Class to hold install and config methods""" + """Class to hold install and config methods""" - def __init__(self, target_host, - initial_manifest=False, - remote_user="root", - home=None, - exec_path=sys.argv[0], - debug=False): + def __init__(self, target_host, + initial_manifest=False, + remote_user="root", + home=None, + exec_path=sys.argv[0], + debug=False): - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user - self.exec_path = exec_path + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.exec_path = exec_path - # FIXME: broken - construct elsewhere! - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + # FIXME: broken - construct elsewhere! + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] - self.path = cdist.path.Path(self.target_host, - initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, - base_dir=home, - debug=debug) - - self.objects_prepared = [] + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] - def cleanup(self): - self.path.cleanup() + def cleanup(self): + self.path.cleanup() - def run_global_explores(self): - """Run global explorers""" - log.info("Running global explorers") - explorers = self.path.list_global_explorers() - if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) + def run_global_explorers(self): + """Run global explorers""" + log.info("Running global explorers") + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) - self.path.transfer_global_explorers() - for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.path.remote_global_explorer_path(explorer)) + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" - type = self.path.get_type_from_object(cdist_object) - self.path.transfer_type_explorers(type) + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) - # Need to transfer at least the parameters for objects to be useful - self.path.transfer_object_parameter(cdist_object) + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) - explorers = self.path.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() - def link_emulator(self): - """Link emulator to types""" - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - self.link_emulator() + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + self.link_emulator() - def run_initial_manifest(self): - """Run the initial manifest""" - log.info("Running initial manifest %s", self.path.initial_manifest) - env = { "__manifest" : self.path.manifest_dir } - self.run_manifest(self.path.initial_manifest, extra_env=env) + def run_initial_manifest(self): + """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.path.get_type_from_object(cdist_object) - manifest = self.path.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.path.object_dir(cdist_object), - "__object_id": self.path.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.path.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - - # Submit debug flag to manifest, can be used by emulator and types - if self.debug: - env['__debug'] = "yes" + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Submit debug flag to manifest, can be used by emulator and types + if self.debug: + env['__debug'] = "yes" - # Required for recording source - env['__cdist_manifest'] = manifest + # Required for recording source + env['__cdist_manifest'] = manifest - # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir - # Other environment stuff - if extra_env: - env.update(extra_env) + # Other environment stuff + if extra_env: + env.update(extra_env) - cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) - if mode == "gencode": - paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) - outfile_fd = open(outfile, "w") + outfile_fd = open(outfile, "w") - # Need to flush to ensure our write is done before stdout write - # FIXME: CODE_HEADER needed in our sh -e scenario???? - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() + # Need to flush to ensure our write is done before stdout write + # FIXME: CODE_HEADER needed in our sh -e scenario? + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() - status = os.stat(outfile) + status = os.stat(outfile) - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - log.info("Running object manifests and type explorers") + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explorers() + self.run_initial_manifest() + + log.info("Running object manifests and type explorers") - old_objects = [] - objects = self.path.list_objects() + old_objects = [] + objects = self.path.list_objects() - # Continue process until no new objects are created anymore - while old_objects != objects: - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) + # Continue process until no new objects are created anymore + while old_objects != objects: + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) - objects = self.path.list_objects() + objects = self.path.list_objects() - def stage_run(self): - """The final (and real) step of deployment""" - log.info("Generating and executing code") - # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") + def stage_run(self): + """The final (and real) step of deployment""" + log.info("Generating and executing code") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() - self.stage_prepare() - self.stage_run() + self.stage_prepare() + self.stage_run() - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() From 2176e4e2d402318713b08a7004317a742effc2b1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 16:05:11 +0200 Subject: [PATCH 29/42] begin new type type (no typo) Signed-off-by: Nico Schottelius --- lib/cdist/type.py | 57 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 lib/cdist/type.py diff --git a/lib/cdist/type.py b/lib/cdist/type.py new file mode 100644 index 00000000..b0280660 --- /dev/null +++ b/lib/cdist/type.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +log = logging.getLogger(__name__) + +class Type(object): + + def __init__(self, path, remote_path): + self.path = path + self.remote_path = remote_path + + def list_explorers(self): + """Return list of available explorers""" + dir = os.path.join(self.path, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.path, "install")) + + # FIXME: Type + def type_dir(self, type, *args): + """Return (sub-)directory of a type""" + return os.path.join(self.type_base_dir, type, *args) + + # FIXME: Type + def remote_type_explorer_dir(self, type): + """Return remote directory that holds the explorers of a type""" + return os.path.join(REMOTE_TYPE_DIR, type, "explorer") From 099adec61fb22ca2b79005612980035326718ebd Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Wed, 5 Oct 2011 16:08:29 +0200 Subject: [PATCH 30/42] start working on object class Signed-off-by: Steven Armstrong --- lib/cdist/object.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 lib/cdist/object.py diff --git a/lib/cdist/object.py b/lib/cdist/object.py new file mode 100644 index 00000000..396d8e48 --- /dev/null +++ b/lib/cdist/object.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import logging +log = logging.getLogger(__name__) + + + +class Object(object): + def __init__(self, path, remote_path): + self.path = path + self.remote_path = remote_path + + From 68889c4bf7a5281cab51a8be0ba7fa7ca40c4da7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 16:12:25 +0200 Subject: [PATCH 31/42] finish type, shrink path Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 31 +------------------------------ lib/cdist/type.py | 10 ++-------- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 5cde357b..e709e6fe 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -171,30 +171,11 @@ class Path: """Return list of available explorers""" return os.listdir(self.global_explorer_dir) - # FIXME: Type - only needs to know its path - def list_type_explorers(self, type): - """Return list of available explorers for a specific type""" - dir = self.type_dir(type, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - # Stays here def list_types(self): """Retuns list of types""" return os.listdir(self.type_base_dir) - # FIXME: type - def is_install_type(self, type): - """Check whether a type is used for installation (if not: for configuration)""" - marker = os.path.join(self.type_dir(type), "install") - return os.path.isfile(marker) - # Stays here def list_object_paths(self, starting_point): """Return list of paths of existing objects""" @@ -260,16 +241,6 @@ class Path: return objects - # FIXME: Type - def type_dir(self, type, *args): - """Return (sub-)directory of a type""" - return os.path.join(self.type_base_dir, type, *args) - - # FIXME: Type - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - # Stays here def transfer_object_parameter(self, cdist_object): """Transfer the object parameter to the remote destination""" @@ -286,7 +257,7 @@ class Path: self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - # Stays here + # Stays here - FIXME: adjust to type code, loop over types! def transfer_type_explorers(self, type): """Transfer explorers of a type, but only once""" if type in self.type_explorers_transferred: diff --git a/lib/cdist/type.py b/lib/cdist/type.py index b0280660..e1c5f589 100644 --- a/lib/cdist/type.py +++ b/lib/cdist/type.py @@ -46,12 +46,6 @@ class Type(object): """Check whether a type is used for installation (if not: for configuration)""" return os.path.isfile(os.path.join(self.path, "install")) - # FIXME: Type - def type_dir(self, type, *args): - """Return (sub-)directory of a type""" - return os.path.join(self.type_base_dir, type, *args) - - # FIXME: Type - def remote_type_explorer_dir(self, type): + def remote_explorer_dir(self): """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + return os.path.join(self.remote_path, "explorer") From a0c984acddbe9fc88c588d3b54901a6a40337faa Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 16:38:19 +0200 Subject: [PATCH 32/42] config+install is broken now due to refactoring Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index a550c45e..8211f374 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -81,6 +81,7 @@ class ConfigInstall: cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() +# FIXME: where to call this from? def run_type_explorer(self, cdist_object): """Run type specific explorers for objects""" @@ -97,6 +98,7 @@ class ConfigInstall: # Need to transfer at least the parameters for objects to be useful self.path.transfer_object_parameter(cdist_object) + # FIXME: Broken due to refactoring into type.py explorers = self.path.list_type_explorers(type) for explorer in explorers: remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] From 2e61e8b202e883a899413bd8c18e9ad04709f031 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Wed, 5 Oct 2011 16:44:22 +0200 Subject: [PATCH 33/42] finish object class Signed-off-by: Steven Armstrong --- lib/cdist/object.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/lib/cdist/object.py b/lib/cdist/object.py index 396d8e48..0a282dc2 100644 --- a/lib/cdist/object.py +++ b/lib/cdist/object.py @@ -26,8 +26,26 @@ log = logging.getLogger(__name__) class Object(object): - def __init__(self, path, remote_path): + + def __init__(self, path, remote_path, object_fq): self.path = path self.remote_path = remote_path + self.object_fq = object_fq + self.type = self.object_fq.split(os.sep)[0] + self.object_id = self.object_fq.split(os.sep)[1:] + self.parameter_dir = os.path.join(self.path, "parameter") + self.remote_object_parameter_dir = os.path.join(self.remote_path, "parameter") + self.object_code_paths = [ + os.path.join(self.path, "code-local"), + os.path.join(self.path, "code-remote")] + @property + def type_explorer_output_dir(self): + """Returns and creates dir of the output for a type explorer""" + if not self.__type_explorer_output_dir: + dir = os.path.join(self.path, "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + self.__type_explorer_output_dir = dir + return self.__type_explorer_output_dir From 7f85aab3ef3b63a261be8da5e9edcc1c39765d2c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 16:45:10 +0200 Subject: [PATCH 34/42] add some pseudo code to adapt to changes Signed-off-by: Nico Schottelius --- lib/cdist/config_install.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/cdist/config_install.py b/lib/cdist/config_install.py index 8211f374..9a84c2cf 100644 --- a/lib/cdist/config_install.py +++ b/lib/cdist/config_install.py @@ -260,6 +260,17 @@ class ConfigInstall: log.debug("Skipping rerun of object %s", cdist_object) continue else: + # FIXME: run_type_explorer: + # object can return type + # type has explorers + # path knows about where to save explorer output + # type = self.path.objects[object].type() + # self.path.types['type'].explorers() + # for explorer in explorers: + # output = cdist.exec.run_debug_or_fail_shell(explorer) + # if output: + # write_output_to(output, os.path.join(self.path.objects[object].explorer_dir(),explorer) ) + # self.run_type_explorer(cdist_object) self.run_type_manifest(cdist_object) self.objects_prepared.append(cdist_object) From f84cabffe35c720614a2d327cc3e3c1d799113b6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Oct 2011 16:49:20 +0200 Subject: [PATCH 35/42] cleanup path Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 54 ++++++++++++++++++++++++++--------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index e709e6fe..5a2b64d2 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -75,6 +75,7 @@ class Path: self.remote_user = remote_user self.remote_prefix = remote_prefix + # Input directories self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") self.cache_dir = os.path.join(self.cache_base_dir, target_host) @@ -83,31 +84,27 @@ class Path: self.manifest_dir = os.path.join(self.conf_dir, "manifest") self.type_base_dir = os.path.join(self.conf_dir, "type") - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) - - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - os.mkdir(self.global_explorer_out_dir) - - self.object_base_dir = os.path.join(self.out_dir, "object") - - # Setup binary directory + contents - self.bin_dir = os.path.join(self.out_dir, "bin") - os.mkdir(self.bin_dir) - - # List of type explorers transferred - self.type_explorers_transferred = {} - - # objects - self.objects_prepared = [] - # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest else: self.initial_manifest = os.path.join(self.manifest_dir, "init") - # FIXME: stays + # Output directories + self.out_dir = os.path.join(self.temp_dir, "out") + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + self.object_base_dir = os.path.join(self.out_dir, "object") + self.bin_dir = os.path.join(self.out_dir, "bin") + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects prepared + self.objects_prepared = [] + + # Create directories + self.__init_out_dirs() + def cleanup(self): # Do not use in __del__: # http://docs.python.org/reference/datamodel.html#customization @@ -120,6 +117,20 @@ class Path: shutil.rmtree(self.cache_dir) shutil.move(self.temp_dir, self.cache_dir) + + def __init_out_dirs(self): + """Initialise output directory structure""" + os.mkdir(self.out_dir) + os.mkdir(self.global_explorer_out_dir) + os.mkdir(self.bin_dir) + + + # Stays here + def list_types(self): + """Retuns list of types""" + return os.listdir(self.type_base_dir) + + ###################################################################### # FIXME: belongs to here - clearify remote* def remote_mkdir(self, directory): @@ -171,11 +182,6 @@ class Path: """Return list of available explorers""" return os.listdir(self.global_explorer_dir) - # Stays here - def list_types(self): - """Retuns list of types""" - return os.listdir(self.type_base_dir) - # Stays here def list_object_paths(self, starting_point): """Return list of paths of existing objects""" From 50a3ad7993d08a6ac630007b49758337d6963fd3 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 00:58:43 +0200 Subject: [PATCH 36/42] proof of concept object oriented aproach Signed-off-by: Steven Armstrong --- oo.py | 242 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 242 insertions(+) create mode 100644 oo.py diff --git a/oo.py b/oo.py new file mode 100644 index 00000000..dd33e0bc --- /dev/null +++ b/oo.py @@ -0,0 +1,242 @@ +import os +import tempfile + +# FIXME: change these to match your environment +os.environ['__cdist_base_dir'] = '/home/sar/vcs/cdist' +# FIXME: testing against the cache, change path +os.environ['__cdist_out_dir'] = '/home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out' + + +''' +cd /path/to/dir/with/this/file +ipython + + +In [1]: import oo + +In [2]: t = oo.Type('__mkfs') + +In [3]: t. +t.base_dir t.is_install t.list_type_names t.name t.path +t.explorers t.is_singleton t.list_types t.optional_parameters t.required_parameters + +In [3]: t.path +Out[3]: '/home/sar/vcs/cdist/conf/type/__mkfs' + +In [4]: t.required_parameters +Out[4]: ['type'] + +In [5]: t.optional_parameters +Out[5]: ['device', 'options', 'blocks'] + +In [6]: t.is +t.is_install t.is_singleton + +In [6]: t.is_singleton +Out[6]: False + +In [7]: o = oo.Object(t, 'dev/sda1') + +In [8]: o. +o.base_dir o.list_object_names o.list_type_names o.parameter o.qualified_name o.type +o.changed o.list_objects o.object_id o.path o.requirements + +In [8]: o.pa +o.parameter o.path + +In [8]: o.path +Out[8]: '/home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/__mkfs/dev/sda1/.cdist' + +In [9]: o.changed +Out[9]: False + +In [10]: o.changed = True + +In [11]: # creates /home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/__mkfs/dev/sda1/.cdist/changed + +In [12]: o.changed +Out[12]: True + +In [13]: o.changed = False + +In [14]: # removes /home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/__mkfs/dev/sda1/.cdist/changed + +In [15]: + +''' + +class Type(object): + + @staticmethod + def base_dir(): + """Return the absolute path to the top level directory where types + are defined. + + Requires the environment variable '__cdist_base_dir' to be set. + + """ + return os.path.join( + os.environ['__cdist_base_dir'], + 'conf', + 'type' + ) + + @classmethod + def list_types(cls): + """Return a list of type instances""" + for type_name in cls.list_type_names(): + yield cls(type_name) + + @classmethod + def list_type_names(cls): + """Return a list of type names""" + return os.listdir(cls.base_dir()) + + + def __init__(self, name): + self.name = name + self.__explorers = None + self.__required_parameters = None + self.__optional_parameters = None + + def __repr__(self): + return '' % self.name + + @property + def path(self): + return os.path.join( + self.base_dir(), + self.name + ) + + @property + def is_singleton(self): + """Check whether a type is a singleton.""" + return os.path.isfile(os.path.join(self.path, "singleton")) + + @property + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.path, "install")) + + @property + def explorers(self): + """Return a list of available explorers""" + if not self.__explorers: + try: + self.__explorers = os.listdir(os.path.join(self.path, "explorer")) + except EnvironmentError as e: + # error ignored + self.__explorers = [] + return self.__explorers + + @property + def required_parameters(self): + """Return a list of required parameters""" + if not self.__required_parameters: + parameters = [] + try: + with open(os.path.join(self.path, "parameter", "required")) as fd: + for line in fd: + parameters.append(line.strip()) + except EnvironmentError as e: + # error ignored + pass + finally: + self.__required_parameters = parameters + return self.__required_parameters + + @property + def optional_parameters(self): + """Return a list of optional parameters""" + if not self.__optional_parameters: + parameters = [] + try: + with open(os.path.join(self.path, "parameter", "optional")) as fd: + for line in fd: + parameters.append(line.strip()) + except EnvironmentError as e: + # error ignored + pass + finally: + self.__optional_parameters = parameters + return self.__optional_parameters + + +class Object(object): + + @staticmethod + def base_dir(): + """Return the absolute path to the top level directory where objects + are defined. + + Requires the environment variable '__cdist_out_dir' to be set. + + """ + base_dir = os.path.join( + os.environ['__cdist_out_dir'], + 'object' + ) + # FIXME: should directory be created elsewhere? + if not os.path.isdir(base_dir): + os.mkdir(base_dir) + return base_dir + + @classmethod + def list_objects(cls): + """Return a list of object instances""" + for object_name in cls.list_object_names(): + type_name = object_name.split(os.sep)[0] + object_id = os.sep.join(object_name.split(os.sep)[1:]) + yield cls(Type(type_name), object_id=object_id) + + @classmethod + def list_type_names(cls): + """Return a list of type names""" + return os.listdir(cls.base_dir()) + + @classmethod + def list_object_names(cls): + """Return a list of object names""" + for path, dirs, files in os.walk(cls.base_dir()): + # FIXME: use constant instead of string + if '.cdist' in dirs: + yield os.path.relpath(path, cls.base_dir()) + + def __init__(self, type, object_id=None, parameter=None, requirements=None): + self.type = type # instance of Type + self.object_id = object_id + self.qualified_name = os.path.join(self.type.name, self.object_id) + self.parameter = parameter or {} + self.requirements = requirements or [] + + def __repr__(self): + return '' % self.qualified_name + + @property + def path(self): + return os.path.join( + self.base_dir(), + self.qualified_name, + '.cdist' + ) + + @property + def changed(self): + """Check whether the object has been changed.""" + return os.path.isfile(os.path.join(self.path, "changed")) + + @changed.setter + def changed(self, value): + """Change the objects changed status.""" + path = os.path.join(self.path, "changed") + if value: + open(path, "w").close() + else: + try: + os.remove(path) + except EnvironmentError: + # ignore + pass + + # FIXME: implement other properties/methods From f262cc4ccf1110365f92ce35c1fc83288ebb8f53 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 01:04:09 +0200 Subject: [PATCH 37/42] ++example usage Signed-off-by: Steven Armstrong --- oo.py | 123 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 123 insertions(+) diff --git a/oo.py b/oo.py index dd33e0bc..081b3e56 100644 --- a/oo.py +++ b/oo.py @@ -63,6 +63,129 @@ In [14]: # removes /home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/ In [15]: + +----- + +In [1]: import oo + +In [2]: oo.Type. +oo.Type.base_dir oo.Type.is_install oo.Type.list_type_names oo.Type.mro oo.Type.path +oo.Type.explorers oo.Type.is_singleton oo.Type.list_types oo.Type.optional_parameters oo.Type.required_parameters + +In [2]: oo.Type.list +oo.Type.list_type_names oo.Type.list_types + +In [2]: oo.Type.list_type_names() +Out[2]: +['__addifnosuchline', + '__apt_ppa', + '__apt_update_index', + '__autofs_map', + '__autofs_master', + '__debconf_set_selections', + '__directory', + '__file', + '__group', + '__issue', + '__key_value', + '__link', + '__mkfs', + '__motd', + '__package', + '__package_apt', + '__package_pacman', + '__package_pkg_openbsd', + '__package_rubygem', + '__package_yum', + '__partition_msdos', + '__partition_msdos_apply', + '__postgres_database', + '__postgres_role', + '__process', + '__removeline', + '__ssh_authorized_key', + '__timezone', + '__user'] + +In [3]: list(oo.Type.list_types()) +Out[3]: +[, + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + , + ] + +In [4]: + +In [4]: oo.Object. +oo.Object.base_dir oo.Object.changed oo.Object.list_object_names oo.Object.list_objects oo.Object.list_type_names oo.Object.mro oo.Object.path + +In [4]: oo.Object.list +oo.Object.list_object_names oo.Object.list_objects oo.Object.list_type_names + +In [4]: oo.Object.list_ob +oo.Object.list_object_names oo.Object.list_objects + +In [4]: oo.Object.list_object_names() +Out[4]: + +In [5]: list(oo.Object.list_object_names()) +Out[5]: +['__mkfs/dev/sda1', + '__mkfs/dev/sda2', + '__mkfs/dev/sda5', + '__mkfs/dev/sda6', + '__mkfs/dev/sda7', + '__partition_msdos/dev/sda1', + '__partition_msdos/dev/sda2', + '__partition_msdos/dev/sda3', + '__partition_msdos/dev/sda5', + '__partition_msdos/dev/sda6', + '__partition_msdos/dev/sda7', + '__partition_msdos_apply/singleton'] + +In [6]: list(oo.Object.list_objects()) +Out[6]: +[, + , + , + , + , + , + , + , + , + , + , + ] + +In [7]: + + ''' class Type(object): From a4abb53ced22a5af16b6f0d68c90146c9e3dd423 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 12:52:13 +0200 Subject: [PATCH 38/42] +MissingEnvironmentVariableError Signed-off-by: Steven Armstrong --- lib/cdist/__init__.py | 10 ++++++++++ oo.py => lib/cdist/core/object.py | 0 2 files changed, 10 insertions(+) rename oo.py => lib/cdist/core/object.py (100%) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index a0ca2ba2..864b4f37 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -24,3 +24,13 @@ VERSION = "2.0.3" class Error(Exception): """Base exception class for this project""" pass + + +class MissingEnvironmentVariableError(Error): + """Raised when a required environment variable is not set.""" + + def __init__(self, name) + self.name = name + + def __str__(self): + return 'Missing required environment variable: {0.name}'.format(o) diff --git a/oo.py b/lib/cdist/core/object.py similarity index 100% rename from oo.py rename to lib/cdist/core/object.py From cdba503ff279a67e0f99ff430952a822dd7e2a88 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 12:57:38 +0200 Subject: [PATCH 39/42] move Object class into its own file Signed-off-by: Steven Armstrong --- lib/cdist/core/object.py | 325 +++++---------------------------------- 1 file changed, 37 insertions(+), 288 deletions(-) diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index 081b3e56..c136390e 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -1,292 +1,37 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + import os -import tempfile -# FIXME: change these to match your environment -os.environ['__cdist_base_dir'] = '/home/sar/vcs/cdist' -# FIXME: testing against the cache, change path -os.environ['__cdist_out_dir'] = '/home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out' - - -''' -cd /path/to/dir/with/this/file -ipython - - -In [1]: import oo - -In [2]: t = oo.Type('__mkfs') - -In [3]: t. -t.base_dir t.is_install t.list_type_names t.name t.path -t.explorers t.is_singleton t.list_types t.optional_parameters t.required_parameters - -In [3]: t.path -Out[3]: '/home/sar/vcs/cdist/conf/type/__mkfs' - -In [4]: t.required_parameters -Out[4]: ['type'] - -In [5]: t.optional_parameters -Out[5]: ['device', 'options', 'blocks'] - -In [6]: t.is -t.is_install t.is_singleton - -In [6]: t.is_singleton -Out[6]: False - -In [7]: o = oo.Object(t, 'dev/sda1') - -In [8]: o. -o.base_dir o.list_object_names o.list_type_names o.parameter o.qualified_name o.type -o.changed o.list_objects o.object_id o.path o.requirements - -In [8]: o.pa -o.parameter o.path - -In [8]: o.path -Out[8]: '/home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/__mkfs/dev/sda1/.cdist' - -In [9]: o.changed -Out[9]: False - -In [10]: o.changed = True - -In [11]: # creates /home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/__mkfs/dev/sda1/.cdist/changed - -In [12]: o.changed -Out[12]: True - -In [13]: o.changed = False - -In [14]: # removes /home/sar/vcs/cdist/cache/sans-asteven-02.ethz.ch/out/object/__mkfs/dev/sda1/.cdist/changed - -In [15]: - - ------ - -In [1]: import oo - -In [2]: oo.Type. -oo.Type.base_dir oo.Type.is_install oo.Type.list_type_names oo.Type.mro oo.Type.path -oo.Type.explorers oo.Type.is_singleton oo.Type.list_types oo.Type.optional_parameters oo.Type.required_parameters - -In [2]: oo.Type.list -oo.Type.list_type_names oo.Type.list_types - -In [2]: oo.Type.list_type_names() -Out[2]: -['__addifnosuchline', - '__apt_ppa', - '__apt_update_index', - '__autofs_map', - '__autofs_master', - '__debconf_set_selections', - '__directory', - '__file', - '__group', - '__issue', - '__key_value', - '__link', - '__mkfs', - '__motd', - '__package', - '__package_apt', - '__package_pacman', - '__package_pkg_openbsd', - '__package_rubygem', - '__package_yum', - '__partition_msdos', - '__partition_msdos_apply', - '__postgres_database', - '__postgres_role', - '__process', - '__removeline', - '__ssh_authorized_key', - '__timezone', - '__user'] - -In [3]: list(oo.Type.list_types()) -Out[3]: -[, - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - , - ] - -In [4]: - -In [4]: oo.Object. -oo.Object.base_dir oo.Object.changed oo.Object.list_object_names oo.Object.list_objects oo.Object.list_type_names oo.Object.mro oo.Object.path - -In [4]: oo.Object.list -oo.Object.list_object_names oo.Object.list_objects oo.Object.list_type_names - -In [4]: oo.Object.list_ob -oo.Object.list_object_names oo.Object.list_objects - -In [4]: oo.Object.list_object_names() -Out[4]: - -In [5]: list(oo.Object.list_object_names()) -Out[5]: -['__mkfs/dev/sda1', - '__mkfs/dev/sda2', - '__mkfs/dev/sda5', - '__mkfs/dev/sda6', - '__mkfs/dev/sda7', - '__partition_msdos/dev/sda1', - '__partition_msdos/dev/sda2', - '__partition_msdos/dev/sda3', - '__partition_msdos/dev/sda5', - '__partition_msdos/dev/sda6', - '__partition_msdos/dev/sda7', - '__partition_msdos_apply/singleton'] - -In [6]: list(oo.Object.list_objects()) -Out[6]: -[, - , - , - , - , - , - , - , - , - , - , - ] - -In [7]: - - -''' - -class Type(object): - - @staticmethod - def base_dir(): - """Return the absolute path to the top level directory where types - are defined. - - Requires the environment variable '__cdist_base_dir' to be set. - - """ - return os.path.join( - os.environ['__cdist_base_dir'], - 'conf', - 'type' - ) - - @classmethod - def list_types(cls): - """Return a list of type instances""" - for type_name in cls.list_type_names(): - yield cls(type_name) - - @classmethod - def list_type_names(cls): - """Return a list of type names""" - return os.listdir(cls.base_dir()) - - - def __init__(self, name): - self.name = name - self.__explorers = None - self.__required_parameters = None - self.__optional_parameters = None - - def __repr__(self): - return '' % self.name - - @property - def path(self): - return os.path.join( - self.base_dir(), - self.name - ) - - @property - def is_singleton(self): - """Check whether a type is a singleton.""" - return os.path.isfile(os.path.join(self.path, "singleton")) - - @property - def is_install(self): - """Check whether a type is used for installation (if not: for configuration)""" - return os.path.isfile(os.path.join(self.path, "install")) - - @property - def explorers(self): - """Return a list of available explorers""" - if not self.__explorers: - try: - self.__explorers = os.listdir(os.path.join(self.path, "explorer")) - except EnvironmentError as e: - # error ignored - self.__explorers = [] - return self.__explorers - - @property - def required_parameters(self): - """Return a list of required parameters""" - if not self.__required_parameters: - parameters = [] - try: - with open(os.path.join(self.path, "parameter", "required")) as fd: - for line in fd: - parameters.append(line.strip()) - except EnvironmentError as e: - # error ignored - pass - finally: - self.__required_parameters = parameters - return self.__required_parameters - - @property - def optional_parameters(self): - """Return a list of optional parameters""" - if not self.__optional_parameters: - parameters = [] - try: - with open(os.path.join(self.path, "parameter", "optional")) as fd: - for line in fd: - parameters.append(line.strip()) - except EnvironmentError as e: - # error ignored - pass - finally: - self.__optional_parameters = parameters - return self.__optional_parameters +import cdist class Object(object): + """Represents a cdist object. + + All interaction with objects in cdist should be done through this class. + Directly accessing an object through the file system from python code is + a bug. + + """ @staticmethod def base_dir(): @@ -296,10 +41,14 @@ class Object(object): Requires the environment variable '__cdist_out_dir' to be set. """ - base_dir = os.path.join( - os.environ['__cdist_out_dir'], - 'object' - ) + try: + base_dir = os.path.join( + os.environ['__cdist_out_dir'], + 'object' + ) + except KeyError as e: + raise cdist.MissingEnvironmentVariableError(e.args[0]) + # FIXME: should directory be created elsewhere? if not os.path.isdir(base_dir): os.mkdir(base_dir) From 080ca10e193a1889f82d3be7f4bcffb352e1a1a8 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 12:58:08 +0200 Subject: [PATCH 40/42] new package cdist.core Signed-off-by: Steven Armstrong --- lib/cdist/core/__init__.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 lib/cdist/core/__init__.py diff --git a/lib/cdist/core/__init__.py b/lib/cdist/core/__init__.py new file mode 100644 index 00000000..80310ffc --- /dev/null +++ b/lib/cdist/core/__init__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +__all__ = ['Type', 'Object'] + +from cdist.core.type import Type +from cdist.core.object import Object From 4de5f6f58ecef4068ccdd2e42f00268fe0693a1f Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 13:00:17 +0200 Subject: [PATCH 41/42] move Type into its own file Signed-off-by: Steven Armstrong --- lib/cdist/core/type.py | 125 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) create mode 100644 lib/cdist/core/type.py diff --git a/lib/cdist/core/type.py b/lib/cdist/core/type.py new file mode 100644 index 00000000..c35e0ad8 --- /dev/null +++ b/lib/cdist/core/type.py @@ -0,0 +1,125 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os + +import cdist + + +class Type(object): + + @staticmethod + def base_dir(): + """Return the absolute path to the top level directory where types + are defined. + + Requires the environment variable '__cdist_base_dir' to be set. + + """ + try: + return os.path.join( + os.environ['__cdist_base_dir'], + 'conf', + 'type' + ) + except KeyError as e: + raise cdist.MissingEnvironmentVariableError(e.args[0]) + + @classmethod + def list_types(cls): + """Return a list of type instances""" + for type_name in cls.list_type_names(): + yield cls(type_name) + + @classmethod + def list_type_names(cls): + """Return a list of type names""" + return os.listdir(cls.base_dir()) + + + def __init__(self, name): + self.name = name + self.__explorers = None + self.__required_parameters = None + self.__optional_parameters = None + + def __repr__(self): + return '' % self.name + + @property + def path(self): + return os.path.join( + self.base_dir(), + self.name + ) + + @property + def is_singleton(self): + """Check whether a type is a singleton.""" + return os.path.isfile(os.path.join(self.path, "singleton")) + + @property + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.path, "install")) + + @property + def explorers(self): + """Return a list of available explorers""" + if not self.__explorers: + try: + self.__explorers = os.listdir(os.path.join(self.path, "explorer")) + except EnvironmentError as e: + # error ignored + self.__explorers = [] + return self.__explorers + + @property + def required_parameters(self): + """Return a list of required parameters""" + if not self.__required_parameters: + parameters = [] + try: + with open(os.path.join(self.path, "parameter", "required")) as fd: + for line in fd: + parameters.append(line.strip()) + except EnvironmentError as e: + # error ignored + pass + finally: + self.__required_parameters = parameters + return self.__required_parameters + + @property + def optional_parameters(self): + """Return a list of optional parameters""" + if not self.__optional_parameters: + parameters = [] + try: + with open(os.path.join(self.path, "parameter", "optional")) as fd: + for line in fd: + parameters.append(line.strip()) + except EnvironmentError as e: + # error ignored + pass + finally: + self.__optional_parameters = parameters + return self.__optional_parameters From 41ab7390dbb223415d15387507a9aaaccb585483 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2011 13:02:08 +0200 Subject: [PATCH 42/42] use constant instead of string for .cdist Signed-off-by: Steven Armstrong --- lib/cdist/core/object.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/cdist/core/object.py b/lib/cdist/core/object.py index c136390e..80c2c351 100644 --- a/lib/cdist/core/object.py +++ b/lib/cdist/core/object.py @@ -22,6 +22,7 @@ import os import cdist +import cdist.path class Object(object): @@ -72,7 +73,7 @@ class Object(object): """Return a list of object names""" for path, dirs, files in os.walk(cls.base_dir()): # FIXME: use constant instead of string - if '.cdist' in dirs: + if cdist.path.DOT_CDIST in dirs: yield os.path.relpath(path, cls.base_dir()) def __init__(self, type, object_id=None, parameter=None, requirements=None): @@ -90,7 +91,7 @@ class Object(object): return os.path.join( self.base_dir(), self.qualified_name, - '.cdist' + cdist.path.DOT_CDIST ) @property