From 20bb4d044b6162bc11cd035829f64e6562f644a5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 13:29:08 +0200 Subject: [PATCH 001/107] --typo: .realines( vs. readlines( Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 5ce947ef..1f44ba2d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -684,7 +684,7 @@ def emulator(): sys.exit(1) else: param_fd = open(file, "r") - param_old = param_fd.realines() + param_old = param_fd.readlines() param_fd.close() if(param_old != param): From e3849b917c29cf6a86241a1fb2b1403e28b5def5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 14:34:19 +0200 Subject: [PATCH 002/107] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 84745512..9e9ca0e7 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -13,6 +13,10 @@ - Fix / rewrite cdist-quickstart +- write tutorial!!!!!!!!! + - like ccollect! + - include ssh control master! + -------------------------------------------------------------------------------- - Initial install support From 143a3a62dda6c7e08c0f3a66dc359507d634744b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 14:40:20 +0200 Subject: [PATCH 003/107] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 9e9ca0e7..44829060 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -16,6 +16,7 @@ - write tutorial!!!!!!!!! - like ccollect! - include ssh control master! + - add local/ hint (and add to git) -------------------------------------------------------------------------------- From 12e6e9288e85fc16f84ccbd3eac73e7cd08ca3e3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Sep 2011 14:45:19 +0200 Subject: [PATCH 004/107] ++tutorial Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 44829060..91dbc72f 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -17,6 +17,9 @@ - like ccollect! - include ssh control master! - add local/ hint (and add to git) + - add hint for ssh StrictHostKeyChecking no + - and that ssh will wait for answer of prompt + - nasty if used in parallel mode (scroll up!) -------------------------------------------------------------------------------- From 462ed49a74e2d8c512b85163d2cd2fafda4f4a09 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 22 Sep 2011 18:43:36 +0200 Subject: [PATCH 005/107] BUGFIX: TypeError: Can't convert 'list' object to str implicitly (in emulator) Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 1f44ba2d..9a745251 100755 --- a/bin/cdist +++ b/bin/cdist @@ -688,8 +688,8 @@ def emulator(): param_fd.close() if(param_old != param): - print("Parameter differs: " + param_old + "vs," + param) - print("Source = " + old_object_source + "new =" + object_source) + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) sys.exit(1) else: param_fd = open(file, "w") From 7a09266abf391a98c3a9f836af4176e6d99c62a9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 13:49:02 +0200 Subject: [PATCH 006/107] allow users to check whether an object changed Signed-off-by: Nico Schottelius --- bin/cdist | 6 +++++- doc/changelog | 4 +++- doc/man/cdist-reference.text.sh | 10 ++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 9a745251..e085a226 100755 --- a/bin/cdist +++ b/bin/cdist @@ -485,7 +485,7 @@ class Cdist: ] for bin in paths: if os.path.isfile(bin): - # omit "gen" from gencode and + # omit "gen" from gencode and use it for output base outfile=os.path.join(self.object_dir(cdist_object), os.path.basename(bin)[3:]) @@ -507,6 +507,10 @@ class Cdist: # Add header and make executable - identically to 0o700 os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + # Mark object as changed + open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() + + if mode == "code": local_dir = self.object_dir(cdist_object) remote_dir = self.remote_object_dir(cdist_object) diff --git a/doc/changelog b/doc/changelog index cc6aa78b..d69011ee 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,7 @@ 2.0.1: - * Bugfix cdist: Always print source of error in case of exec errors + * Bugfix core: Always print source of error in case of exec errors + * Bugfix core: Various smaller bugs in string concatenation + * Feature: Add marker "changed" to changed objects 2.0.0: 2011-09-16 * New Type: __package_rubygem (Chase Allen James) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index e38f157d..c205bdcc 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -154,6 +154,16 @@ done cat << eof +OBJECTS +------- +For object to object communication and tests, the following paths are +usable within a object directory: + +changed:: + This empty file exists in an object directory, if the object has + code to be excuted (either remote or local) + + ENVIRONMENT VARIABLES --------------------- __explorer:: From ca3644b73a4ceb9ac0c156ca07d1b2b49851675d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 13:50:49 +0200 Subject: [PATCH 007/107] ++version Signed-off-by: Nico Schottelius --- doc/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index d69011ee..b1149eb1 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,4 +1,4 @@ -2.0.1: +2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors * Bugfix core: Various smaller bugs in string concatenation * Feature: Add marker "changed" to changed objects From 412778206c15d929f26c567b36bb85557818cef1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 14:21:33 +0200 Subject: [PATCH 008/107] and increment version Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index e085a226..30db1ec9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -57,7 +57,7 @@ REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" TYPE_PREFIX = "__" -VERSION = "2.0.0" +VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() From 199245e6ce100718c47a40839930dd55ca1c9d21 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 14:54:39 +0200 Subject: [PATCH 009/107] ignore more pycache stuff Signed-off-by: Nico Schottelius --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d606aec7..259e3be3 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ doc/man/man*/docbook-xsl.css cache/ # Python -bin/__pycache__/ +*/__pycache__/ From b8ff4c9609d61033deab83b4e3be082cde7b45fe Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 14:55:00 +0200 Subject: [PATCH 010/107] begin split into smaller files Signed-off-by: Nico Schottelius --- bin/cdist | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/cdist b/bin/cdist index 30db1ec9..fe3c17fd 100755 --- a/bin/cdist +++ b/bin/cdist @@ -62,6 +62,10 @@ VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() +# Begin to split into parts +sys.path.insert(0, + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib/python'))) def file_to_list(filename): """Return list from \n seperated file""" From 211212e079fb1582658b4448e0c1ac2c0efdf0d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 16:01:24 +0200 Subject: [PATCH 011/107] test template Signed-off-by: Nico Schottelius --- test/cdist.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 test/cdist.py diff --git a/test/cdist.py b/test/cdist.py new file mode 100644 index 00000000..3ccc69bc --- /dev/null +++ b/test/cdist.py @@ -0,0 +1,12 @@ +import cdist +import unittest + + +class CdistGeneric(unittest.TestCase): + + def test_initial_manifest(self): + self.assertEqual(numeral, result) + + +if __name__ == '__main__': + unittest.main() From ad5c33b7467e8224997183d48994b3b0de0ac9c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 16:38:47 +0200 Subject: [PATCH 012/107] %s/\( \)/\1 /g + fix logo Signed-off-by: Nico Schottelius --- bin/cdist | 1348 ++++++++++++++++++++++++++--------------------------- 1 file changed, 674 insertions(+), 674 deletions(-) diff --git a/bin/cdist b/bin/cdist index fe3c17fd..2100bd4d 100755 --- a/bin/cdist +++ b/bin/cdist @@ -48,745 +48,745 @@ BANNER = """ """ # Given paths from installation -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") -CODE_HEADER = "#!/bin/sh -e\n" -DOT_CDIST = ".cdist" -TYPE_PREFIX = "__" -VERSION = "2.0.1" +CODE_HEADER = "#!/bin/sh -e\n" +DOT_CDIST = ".cdist" +TYPE_PREFIX = "__" +VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() # Begin to split into parts sys.path.insert(0, - os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib/python'))) + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib/python'))) def file_to_list(filename): - """Return list from \n seperated file""" - if os.path.isfile(filename): - file_fd = open(filename, "r") - lines = file_fd.readlines() - file_fd.close() + """Return list from \n seperated file""" + if os.path.isfile(filename): + file_fd = open(filename, "r") + lines = file_fd.readlines() + file_fd.close() - # Remove \n from all lines - lines = map(lambda s: s.strip(), lines) - else: - lines = [] + # Remove \n from all lines + lines = map(lambda s: s.strip(), lines) + else: + lines = [] - return lines + return lines def exit_error(*args): - log.error(*args) - sys.exit(1) + log.error(*args) + sys.exit(1) class Cdist: - """Cdist main class to hold arbitrary data""" - - def __init__(self, target_host, - initial_manifest=False, remote_user="root", - home=None, debug=False): - self.target_host = target_host - self.remote_prefix = ["ssh", "root@" + self.target_host] - - # Setup directory paths - self.temp_dir = tempfile.mkdtemp() - - self.debug = debug - - if home: - self.base_dir = home - else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) - self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") - - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) - - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - os.mkdir(self.global_explorer_out_dir) - - self.object_base_dir = os.path.join(self.out_dir, "object") - - # Setup binary directory + contents - self.bin_dir = os.path.join(self.out_dir, "bin") - os.mkdir(self.bin_dir) - self.link_type_to_emulator() - - # List of type explorers transferred - self.type_explorers_transferred = {} - - # objects - self.objects_prepared = [] - - self.remote_user = remote_user - - # Mostly static, but can be overwritten on user demand - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_dir, "init") - - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) - # Remove previous cache - if os.path.exists(self.cache_dir): - shutil.rmtree(self.cache_dir) - shutil.move(self.temp_dir, self.cache_dir) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) - - def remote_cat(filename): - """Use cat on the remote side for output""" - self.run_or_fail(["cat", filename], remote=True) - - def shell_run_or_debug_fail(self, script, *args, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - remote = true - - del kargs["remote"] - - log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - if remote: - remote_cat(script) - else: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - - exit_error("Command failed (shell): " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - - del kargs["remote"] - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - exit_error("Command failed: " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - - def remove_remote_dir(self, destination): - self.run_or_fail(["rm", "-rf", destination], remote=True) - - def transfer_dir(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - - def transfer_file(self, source, destination): - """Transfer file""" - self.run_or_fail(["scp", "-q", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - - def global_explorer_output_path(self, explorer): - """Returns path of the output for a global explorer""" - return os.path.join(self.global_explorer_out_dir, explorer) - - def type_explorer_output_dir(self, cdist_object): - """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_dir(cdist_object), "explorer") - if not os.path.isdir(dir): - os.mkdir(dir) - - return dir - - def remote_global_explorer_path(self, explorer): - """Returns path to the remote explorer""" - return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) - - def list_global_explorers(self): - """Return list of available explorers""" - return os.listdir(self.global_explorer_dir) - - def list_type_explorers(self, type): - """Return list of available explorers for a specific type""" - dir = self.type_dir(type, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - - def list_types(self): - return os.listdir(self.type_base_dir) - - def list_object_paths(self, starting_point): - """Return list of paths of existing objects""" - object_paths = [] - - for content in os.listdir(starting_point): - full_path = os.path.join(starting_point, content) - if os.path.isdir(full_path): - object_paths.extend(self.list_object_paths(starting_point = full_path)) - - # Directory contains .cdist -> is an object - if content == DOT_CDIST: - object_paths.append(starting_point) - - return object_paths - - def get_type_from_object(self, cdist_object): - """Returns the first part (i.e. type) of an object""" - return cdist_object.split(os.sep)[0] - - def get_object_id_from_object(self, cdist_object): - """Returns everything but the first part (i.e. object_id) of an object""" - return os.sep.join(cdist_object.split(os.sep)[1:]) - - def object_dir(self, cdist_object): - """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) - - def remote_object_dir(self, cdist_object): - """Returns the remote full path to the object (including .cdist)""" - return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) - - def object_parameter_dir(self, cdist_object): - """Returns the dir to the object parameter""" - return os.path.join(self.object_dir(cdist_object), "parameter") - - def remote_object_parameter_dir(self, cdist_object): - """Returns the remote dir to the object parameter""" - return os.path.join(self.remote_object_dir(cdist_object), "parameter") - - def object_code_paths(self, cdist_object): - """Return paths to code scripts of object""" - return [os.path.join(self.object_dir(cdist_object), "code-local"), - os.path.join(self.object_dir(cdist_object), "code-remote")] - - def list_objects(self): - """Return list of existing objects""" - - objects = [] - if os.path.isdir(self.object_base_dir): - object_paths = self.list_object_paths(self.object_base_dir) - - for path in object_paths: - objects.append(os.path.relpath(path, self.object_base_dir)) - - return objects - - def type_dir(self, type, *args): - """Return directory the type""" - return os.path.join(self.type_base_dir, type, *args) - - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - - def transfer_global_explorers(self): - """Transfer the global explorers""" - self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) - self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - - def transfer_type_explorers(self, type): - """Transfer explorers of a type, but only once""" - if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for explorers of %s", type) - return - else: - # Do not retransfer - self.type_explorers_transferred[type] = 1 - - src = self.type_dir(type, "explorer") - remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.remote_type_explorer_dir(type) - - # Only continue, if there is at least the directory - if os.path.isdir(src): - # Ensure that the path exists - self.remote_mkdir(remote_base) - self.transfer_dir(src, dst) - - - def link_type_to_emulator(self): - """Link type names to cdist-type-emulator""" - source = os.path.abspath(sys.argv[0]) - for type in self.list_types(): - destination = os.path.join(self.bin_dir, type) - log.debug("Linking %s to %s", source, destination) - os.symlink(source, destination) - - def run_global_explores(self): - """Run global explorers""" - explorers = self.list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) - - self.transfer_global_explorers() - for explorer in explorers: - output = self.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.remote_global_explorer_path(explorer)) - - self.run_or_fail(cmd, stdout=output_fd, remote=True) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - # Based on bin/cdist-object-explorer-run - - # Transfering explorers for this type - type = self.get_type_from_object(cdist_object) - self.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.transfer_object_parameter(cdist_object) - - explorers = self.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.manifest_dir } - self.run_manifest(self.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.get_type_from_object(cdist_object) - manifest = self.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), - "__object_id": self.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") - env['__cdist_local_base_dir'] = self.temp_dir - - # Submit information to new type emulator - env['__cdist_manifest'] = manifest - env['__cdist_type_base_dir'] = self.type_base_dir - - # Other environment stuff - if extra_env: - env.update(extra_env) - - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.object_dir(cdist_object), "require") - requirements = file_to_list(file) - type = self.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object) - env["__object_id"] = self.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(type) - - if mode == "gencode": - paths = [ - self.type_dir(type, "gencode-local"), - self.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.object_dir(cdist_object) - remote_dir = self.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - self.run_or_fail([bin], remote=False) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.transfer_file(local_remote_code, remote_remote_code) - self.run_or_fail([remote_remote_code], remote=True) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue + """Cdist main class to hold arbitrary data""" + + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + home=None, debug=False): + self.target_host = target_host + self.remote_prefix = ["ssh", "root@" + self.target_host] + + # Setup directory paths + self.temp_dir = tempfile.mkdtemp() + + self.debug = debug + + if home: + self.base_dir = home + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + self.object_base_dir = os.path.join(self.out_dir, "object") + + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + self.link_type_to_emulator() + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects + self.objects_prepared = [] + + self.remote_user = remote_user + + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_dir, "init") + + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + shutil.move(self.temp_dir, self.cache_dir) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + self.nun_or_fail(["mkdir", "-p", directory], remote=True) + + def remote_cat(filename): + """Use cat on the remote side for output""" + self.run_or_fail(["cat", filename], remote=True) + + def shell_run_or_debug_fail(self, script, *args, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + remote = False + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) + script_fd = open(script) + print(script_fd.read()) + script_fd.close() - objects = self.list_objects() + exit_error("Command failed (shell): " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") + def run_or_fail(self, *args, **kargs): + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() + del kargs["remote"] - self.stage_prepare() - self.stage_run() + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + exit_error("Command failed: " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() + def remove_remote_dir(self, destination): + self.run_or_fail(["rm", "-rf", destination], remote=True) + + def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_dir(destination) + self.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + self.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_dir(cdist_object), "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_dir) + + def list_type_explorers(self, type): + """Return list of available explorers for a specific type""" + dir = self.type_dir(type, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def list_types(self): + return os.listdir(self.type_base_dir) + + def list_object_paths(self, starting_point): + """Return list of paths of existing objects""" + object_paths = [] + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + if os.path.isdir(full_path): + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == DOT_CDIST: + object_paths.append(starting_point) + + return object_paths + + def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" + return cdist_object.split(os.sep)[0] + + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(self.object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(self.remote_object_dir(cdist_object), "parameter") + + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] + + def list_objects(self): + """Return list of existing objects""" + + objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) + + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) + + return objects + + def type_dir(self, type, *args): + """Return directory the type""" + return os.path.join(self.type_base_dir, type, *args) + + def remote_type_explorer_dir(self, type): + """Return remote directory that holds the explorers of a type""" + return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + + def transfer_object_parameter(self, cdist_object): + """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + + def transfer_global_explorers(self): + """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if type in self.type_explorers_transferred: + log.debug("Skipping retransfer for explorers of %s", type) + return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 + + src = self.type_dir(type, "explorer") + remote_base = os.path.join(REMOTE_TYPE_DIR, type) + dst = self.remote_type_explorer_dir(type) + + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_mkdir(remote_base) + self.transfer_dir(src, dst) + + + def link_type_to_emulator(self): + """Link type names to cdist-type-emulator""" + source = os.path.abspath(sys.argv[0]) + for type in self.list_types(): + destination = os.path.join(self.bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) + + def run_global_explores(self): + """Run global explorers""" + explorers = self.list_global_explorers() + if(len(explorers) == 0): + exit_error("No explorers found in", self.global_explorer_dir) + + self.transfer_global_explorers() + for explorer in explorers: + output = self.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.remote_global_explorer_path(explorer)) + + self.run_or_fail(cmd, stdout=output_fd, remote=True) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + # Based on bin/cdist-object-explorer-run + + # Transfering explorers for this type + type = self.get_type_from_object(cdist_object) + self.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.transfer_object_parameter(cdist_object) + + explorers = self.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) + output_fd.close() + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.remove_remote_dir(REMOTE_BASE_DIR) + self.remote_mkdir(REMOTE_BASE_DIR) + + def run_initial_manifest(self): + """Run the initial manifest""" + env = { "__manifest" : self.manifest_dir } + self.run_manifest(self.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.get_type_from_object(cdist_object) + manifest = self.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.object_dir(cdist_object), + "__object_id": self.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") + env['__cdist_local_base_dir'] = self.temp_dir + + # Submit information to new type emulator + env['__cdist_manifest'] = manifest + env['__cdist_type_base_dir'] = self.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.object_dir(cdist_object), "require") + requirements = file_to_list(file) + type = self.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + env["__object"] = self.object_dir(cdist_object) + env["__object_id"] = self.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.type_dir(type) + + if mode == "gencode": + paths = [ + self.type_dir(type, "gencode-local"), + self.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.object_dir(cdist_object) + remote_dir = self.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + self.run_or_fail([bin], remote=False) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.transfer_file(local_remote_code, remote_remote_code) + self.run_or_fail([remote_remote_code], remote=True) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + old_objects = [] + objects = self.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.debug("Actual run objects") + # Now do the final steps over the existing objects + for cdist_object in self.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() def banner(args): - """Guess what :-)""" - print(BANNER) - sys.exit(0) + """Guess what :-)""" + print(BANNER) + sys.exit(0) def config(args): - """Configure remote system""" - process = {} + """Configure remote system""" + process = {} - time_start = datetime.datetime.now() + time_start = datetime.datetime.now() - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() + for host in args.host: + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) def install(args): - """Install remote system""" - process = {} + """Install remote system""" + process = {} def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(sys.argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] - parser = argparse.ArgumentParser(add_help=False) + parser = argparse.ArgumentParser(add_help=False) - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) + # Setup optional parameters + for parameter in file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) + # Setup required parameters + for parameter in file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) - # Setup positional parameter, if not singleton + # Setup positional parameter, if not singleton - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) + # And finally verify parameter + args = parser.parse_args(sys.argv[1:]) - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] - # FIXME: verify object id - log.debug(args) + # FIXME: verify object id + log.debug(args) - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") + object_dir = os.path.join(global_dir, "object", type, + object_id, DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") - object_source_file = os.path.join(object_dir, "source") + object_source_file = os.path.join(object_dir, "source") - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + exit_error(param_out_dir + ": " + error.args[1]) - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + print("New parameter + " + param + "specified, aborting") + print("Source = " + old_object_source + "new =" + object_source) + sys.exit(1) + else: + param_fd = open(file, "r") + param_old = param_fd.readlines() + param_fd.close() + + if(param_old != param): + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) + sys.exit(1) else: - param_fd = open(file, "r") - param_old = param_fd.readlines() - param_fd.close() - - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + print(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) + # sys.exit(1) + print("Finished " + type + "/" + object_id + repr(params)) def commandline(): - """Parse command line""" - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true') - # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) - parser['main'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") - # Banner - parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) - parser['banner'].set_defaults(func=banner) + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) + parser['banner'].set_defaults(func=banner) - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) + parser['config'].set_defaults(func=config) - # Install - parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) - for p in parser: - parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" - args = parser['main'].parse_args(sys.argv[1:]) + args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) + # Most subcommands have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) - args.func(args) + args.func(args) if __name__ == "__main__": - try: - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() - else: - commandline() - except KeyboardInterrupt: - sys.exit(0) + try: + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + emulator() + else: + commandline() + except KeyboardInterrupt: + sys.exit(0) From f4da3b96d8a6fc97b241b43eb0c2520798507926 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 16:41:55 +0200 Subject: [PATCH 013/107] fix indentions Signed-off-by: Nico Schottelius --- bin/cdist | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/bin/cdist b/bin/cdist index 2100bd4d..f3b88fdd 100755 --- a/bin/cdist +++ b/bin/cdist @@ -48,16 +48,16 @@ BANNER = """ """ # Given paths from installation -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") -REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") CODE_HEADER = "#!/bin/sh -e\n" -DOT_CDIST = ".cdist" +DOT_CDIST = ".cdist" TYPE_PREFIX = "__" -VERSION = "2.0.1" +VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() @@ -104,13 +104,13 @@ class Cdist: else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") self.out_dir = os.path.join(self.temp_dir, "out") os.mkdir(self.out_dir) From ea9dc8d60c648448c42f765fcab265c33cd55c0a Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 23 Sep 2011 16:43:25 +0200 Subject: [PATCH 014/107] handle errors with exceptions instead of function Signed-off-by: Steven Armstrong --- bin/cdist | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/bin/cdist b/bin/cdist index fe3c17fd..e3d63458 100755 --- a/bin/cdist +++ b/bin/cdist @@ -81,9 +81,9 @@ def file_to_list(filename): return lines -def exit_error(*args): - log.error(*args) - sys.exit(1) +class CdistError(Exception): + """Base exception class for this project""" + pass class Cdist: """Cdist main class to hold arbitrary data""" @@ -185,9 +185,9 @@ class Cdist: print(script_fd.read()) script_fd.close() - exit_error("Command failed (shell): " + " ".join(*args)) + raise CdistError("Command failed (shell): " + " ".join(*args)) except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) + raise CdistError(" ".join(*args) + ": " + error.args[1]) def run_or_fail(self, *args, **kargs): if "remote" in kargs: @@ -200,9 +200,9 @@ class Cdist: try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - exit_error("Command failed: " + " ".join(*args)) + raise CdistError("Command failed: " + " ".join(*args)) except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) + raise CdistError(" ".join(*args) + ": " + error.args[1]) def remove_remote_dir(self, destination): @@ -368,7 +368,7 @@ class Cdist: """Run global explorers""" explorers = self.list_global_explorers() if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) + raise CdistError("No explorers found in", self.global_explorer_dir) self.transfer_global_explorers() for explorer in explorers: @@ -674,7 +674,7 @@ def emulator(): try: os.makedirs(param_out_dir, exist_ok=True) except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) + raise CdistError(param_out_dir + ": " + error.args[1]) # Record parameter params = vars(args) @@ -790,3 +790,6 @@ if __name__ == "__main__": commandline() except KeyboardInterrupt: sys.exit(0) + except CdistError as e: + log.error(e) + sys.exit(1) From b72fab3c01804b19f00744321f8e8e854e10c290 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:00:48 +0200 Subject: [PATCH 015/107] begin split into path module Signed-off-by: Nico Schottelius --- module/cdist/__init__.py | 0 module/cdist/path.py | 11 +++++++++++ 2 files changed, 11 insertions(+) create mode 100644 module/cdist/__init__.py create mode 100644 module/cdist/path.py diff --git a/module/cdist/__init__.py b/module/cdist/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/module/cdist/path.py b/module/cdist/path.py new file mode 100644 index 00000000..37d92940 --- /dev/null +++ b/module/cdist/path.py @@ -0,0 +1,11 @@ +class Path: + """Class that handles path related configurations""" + + def __init__(self, home=None): + if home: + self.base_dir = home + else: + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + + + print("Base:" + self.base_dir) From 3ace43ff01528717528034bb827ccc03b056b715 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:00:59 +0200 Subject: [PATCH 016/107] ignore pycache everywhere Signed-off-by: Nico Schottelius --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 259e3be3..bd4ecaaa 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,4 @@ doc/man/man*/docbook-xsl.css cache/ # Python -*/__pycache__/ +**/__pycache__/ From a2f3246758cf954e1eecf682c16992cbf9a6026d Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 23 Sep 2011 17:03:00 +0200 Subject: [PATCH 017/107] fix typo /nun_or_fail/run_or_fail/ Signed-off-by: Steven Armstrong --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 0aa661a3..fde7a251 100755 --- a/bin/cdist +++ b/bin/cdist @@ -153,7 +153,7 @@ class Cdist: def remote_mkdir(self, directory): """Create directory on remote side""" - self.nun_or_fail(["mkdir", "-p", directory], remote=True) + self.run_or_fail(["mkdir", "-p", directory], remote=True) def remote_cat(filename): """Use cat on the remote side for output""" From 747517f06782db10da115f3f66f2115cf7e4fa48 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:06:42 +0200 Subject: [PATCH 018/107] import path information from current cdist Signed-off-by: Nico Schottelius --- module/cdist/path.py | 56 ++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 52 insertions(+), 4 deletions(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index 37d92940..ee492cc5 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -1,11 +1,59 @@ class Path: """Class that handles path related configurations""" - def __init__(self, home=None): + def __init__(self, target_host, base_dir=None): + # Base and Temp Base if home: - self.base_dir = home + self.base_dir = base_dir else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - print("Base:" + self.base_dir) + self.temp_dir = tempfile.mkdtemp() + + self.conf_dir = os.path.join(self.base_dir, "conf") + self.cache_base_dir = os.path.join(self.base_dir, "cache") + self.cache_dir = os.path.join(self.cache_base_dir, target_host) + self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") + self.lib_dir = os.path.join(self.base_dir, "lib") + self.manifest_dir = os.path.join(self.conf_dir, "manifest") + self.type_base_dir = os.path.join(self.conf_dir, "type") + + self.out_dir = os.path.join(self.temp_dir, "out") + os.mkdir(self.out_dir) + + self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") + os.mkdir(self.global_explorer_out_dir) + + self.object_base_dir = os.path.join(self.out_dir, "object") + + # Setup binary directory + contents + self.bin_dir = os.path.join(self.out_dir, "bin") + os.mkdir(self.bin_dir) + self.link_type_to_emulator() + + # List of type explorers transferred + self.type_explorers_transferred = {} + + # objects + self.objects_prepared = [] + + self.remote_user = remote_user + + # Mostly static, but can be overwritten on user demand + if initial_manifest: + self.initial_manifest = initial_manifest + else: + self.initial_manifest = os.path.join(self.manifest_dir, "init") + + def cleanup(self): + # Do not use in __del__: + # http://docs.python.org/reference/datamodel.html#customization + # "other globals referenced by the __del__() method may already have been deleted + # or in the process of being torn down (e.g. the import machinery shutting down)" + # + log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) + # Remove previous cache + if os.path.exists(self.cache_dir): + shutil.rmtree(self.cache_dir) + shutil.move(self.temp_dir, self.cache_dir) + From 26772939ca8dce14507358f100cbfd9bdeacdb4d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:11:03 +0200 Subject: [PATCH 019/107] import whole cdist into path.py and strip down (most stuff is moved here anyway) Signed-off-by: Nico Schottelius --- module/cdist/path.py | 665 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 665 insertions(+) diff --git a/module/cdist/path.py b/module/cdist/path.py index ee492cc5..387481ca 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -1,3 +1,28 @@ +# Given paths from installation +REMOTE_BASE_DIR = "/var/lib/cdist" +REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") +REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") +REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") +REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") + +CODE_HEADER = "#!/bin/sh -e\n" +DOT_CDIST = ".cdist" +TYPE_PREFIX = "__" + +def file_to_list(filename): + """Return list from \n seperated file""" + if os.path.isfile(filename): + file_fd = open(filename, "r") + lines = file_fd.readlines() + file_fd.close() + + # Remove \n from all lines + lines = map(lambda s: s.strip(), lines) + else: + lines = [] + + return lines + class Path: """Class that handles path related configurations""" @@ -57,3 +82,643 @@ class Path: shutil.rmtree(self.cache_dir) shutil.move(self.temp_dir, self.cache_dir) + + def remote_mkdir(self, directory): + """Create directory on remote side""" + self.run_or_fail(["mkdir", "-p", directory], remote=True) + + def remote_cat(filename): + """Use cat on the remote side for output""" + self.run_or_fail(["cat", filename], remote=True) + + def shell_run_or_debug_fail(self, script, *args, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + remote = False + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) + else: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + + exit_error("Command failed (shell): " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) + + def run_or_fail(self, *args, **kargs): + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + + del kargs["remote"] + + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + exit_error("Command failed: " + " ".join(*args)) + except OSError as error: + exit_error(" ".join(*args) + ": " + error.args[1]) + + + def remove_remote_dir(self, destination): + self.run_or_fail(["rm", "-rf", destination], remote=True) + + def transfer_dir(self, source, destination): + """Transfer directory and previously delete the remote destination""" + self.remove_remote_dir(destination) + self.run_or_fail(["scp", "-qr", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def transfer_file(self, source, destination): + """Transfer file""" + self.run_or_fail(["scp", "-q", source, + self.remote_user + "@" + + self.target_host + ":" + + destination]) + + def global_explorer_output_path(self, explorer): + """Returns path of the output for a global explorer""" + return os.path.join(self.global_explorer_out_dir, explorer) + + def type_explorer_output_dir(self, cdist_object): + """Returns and creates dir of the output for a type explorer""" + dir = os.path.join(self.object_dir(cdist_object), "explorer") + if not os.path.isdir(dir): + os.mkdir(dir) + + return dir + + def remote_global_explorer_path(self, explorer): + """Returns path to the remote explorer""" + return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) + + def list_global_explorers(self): + """Return list of available explorers""" + return os.listdir(self.global_explorer_dir) + + def list_type_explorers(self, type): + """Return list of available explorers for a specific type""" + dir = self.type_dir(type, "explorer") + if os.path.isdir(dir): + list = os.listdir(dir) + else: + list = [] + + log.debug("Explorers for %s in %s: %s", type, dir, list) + + return list + + def list_types(self): + return os.listdir(self.type_base_dir) + + def list_object_paths(self, starting_point): + """Return list of paths of existing objects""" + object_paths = [] + + for content in os.listdir(starting_point): + full_path = os.path.join(starting_point, content) + if os.path.isdir(full_path): + object_paths.extend(self.list_object_paths(starting_point = full_path)) + + # Directory contains .cdist -> is an object + if content == DOT_CDIST: + object_paths.append(starting_point) + + return object_paths + + def get_type_from_object(self, cdist_object): + """Returns the first part (i.e. type) of an object""" + return cdist_object.split(os.sep)[0] + + def get_object_id_from_object(self, cdist_object): + """Returns everything but the first part (i.e. object_id) of an object""" + return os.sep.join(cdist_object.split(os.sep)[1:]) + + def object_dir(self, cdist_object): + """Returns the full path to the object (including .cdist)""" + return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) + + def remote_object_dir(self, cdist_object): + """Returns the remote full path to the object (including .cdist)""" + return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) + + def object_parameter_dir(self, cdist_object): + """Returns the dir to the object parameter""" + return os.path.join(self.object_dir(cdist_object), "parameter") + + def remote_object_parameter_dir(self, cdist_object): + """Returns the remote dir to the object parameter""" + return os.path.join(self.remote_object_dir(cdist_object), "parameter") + + def object_code_paths(self, cdist_object): + """Return paths to code scripts of object""" + return [os.path.join(self.object_dir(cdist_object), "code-local"), + os.path.join(self.object_dir(cdist_object), "code-remote")] + + def list_objects(self): + """Return list of existing objects""" + + objects = [] + if os.path.isdir(self.object_base_dir): + object_paths = self.list_object_paths(self.object_base_dir) + + for path in object_paths: + objects.append(os.path.relpath(path, self.object_base_dir)) + + return objects + + def type_dir(self, type, *args): + """Return directory the type""" + return os.path.join(self.type_base_dir, type, *args) + + def remote_type_explorer_dir(self, type): + """Return remote directory that holds the explorers of a type""" + return os.path.join(REMOTE_TYPE_DIR, type, "explorer") + + def transfer_object_parameter(self, cdist_object): + """Transfer the object parameter to the remote destination""" + # Create base path before using mkdir -p + self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) + + # Synchronise parameter dir afterwards + self.transfer_dir(self.object_parameter_dir(cdist_object), + self.remote_object_parameter_dir(cdist_object)) + + def transfer_global_explorers(self): + """Transfer the global explorers""" + self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) + self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) + + def transfer_type_explorers(self, type): + """Transfer explorers of a type, but only once""" + if type in self.type_explorers_transferred: + log.debug("Skipping retransfer for explorers of %s", type) + return + else: + # Do not retransfer + self.type_explorers_transferred[type] = 1 + + src = self.type_dir(type, "explorer") + remote_base = os.path.join(REMOTE_TYPE_DIR, type) + dst = self.remote_type_explorer_dir(type) + + # Only continue, if there is at least the directory + if os.path.isdir(src): + # Ensure that the path exists + self.remote_mkdir(remote_base) + self.transfer_dir(src, dst) + + + def link_type_to_emulator(self): + """Link type names to cdist-type-emulator""" + source = os.path.abspath(sys.argv[0]) + for type in self.list_types(): + destination = os.path.join(self.bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) + + def run_global_explores(self): + """Run global explorers""" + explorers = self.list_global_explorers() + if(len(explorers) == 0): + exit_error("No explorers found in", self.global_explorer_dir) + + self.transfer_global_explorers() + for explorer in explorers: + output = self.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.remote_global_explorer_path(explorer)) + + self.run_or_fail(cmd, stdout=output_fd, remote=True) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + # Based on bin/cdist-object-explorer-run + + # Transfering explorers for this type + type = self.get_type_from_object(cdist_object) + self.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.transfer_object_parameter(cdist_object) + + explorers = self.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) + output_fd.close() + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.remove_remote_dir(REMOTE_BASE_DIR) + self.remote_mkdir(REMOTE_BASE_DIR) + + def run_initial_manifest(self): + """Run the initial manifest""" + env = { "__manifest" : self.manifest_dir } + self.run_manifest(self.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.get_type_from_object(cdist_object) + manifest = self.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.object_dir(cdist_object), + "__object_id": self.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") + env['__cdist_local_base_dir'] = self.temp_dir + + # Submit information to new type emulator + env['__cdist_manifest'] = manifest + env['__cdist_type_base_dir'] = self.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + self.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.object_dir(cdist_object), "require") + requirements = file_to_list(file) + type = self.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.out_dir + env["__object"] = self.object_dir(cdist_object) + env["__object_id"] = self.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.type_dir(type) + + if mode == "gencode": + paths = [ + self.type_dir(type, "gencode-local"), + self.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.object_dir(cdist_object) + remote_dir = self.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + self.run_or_fail([bin], remote=False) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.transfer_file(local_remote_code, remote_remote_code) + self.run_or_fail([remote_remote_code], remote=True) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + old_objects = [] + objects = self.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.debug("Actual run objects") + # Now do the final steps over the existing objects + for cdist_object in self.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() + +def banner(args): + """Guess what :-)""" + print(BANNER) + sys.exit(0) + +def config(args): + """Configure remote system""" + process = {} + + time_start = datetime.datetime.now() + + for host in args.host: + c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() + + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) + +def install(args): + """Install remote system""" + process = {} + +def emulator(): + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(sys.argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] + + parser = argparse.ArgumentParser(add_help=False) + + # Setup optional parameters + for parameter in file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) + + # Setup required parameters + for parameter in file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) + + # Setup positional parameter, if not singleton + + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) + + # And finally verify parameter + args = parser.parse_args(sys.argv[1:]) + + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id + + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] + + # FIXME: verify object id + log.debug(args) + + object_dir = os.path.join(global_dir, "object", type, + object_id, DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") + + object_source_file = os.path.join(object_dir, "source") + + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() + + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + exit_error(param_out_dir + ": " + error.args[1]) + + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) + + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + print("New parameter + " + param + "specified, aborting") + print("Source = " + old_object_source + "new =" + object_source) + sys.exit(1) + else: + param_fd = open(file, "r") + param_old = param_fd.readlines() + param_fd.close() + + if(param_old != param): + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) + sys.exit(1) + else: + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() + + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + print(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() + + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() + + # sys.exit(1) + print("Finished " + type + "/" + object_id + repr(params)) + + +def commandline(): + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true') + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") + + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) + parser['banner'].set_defaults(func=banner) + + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) + parser['config'].set_defaults(func=config) + + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) + + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + + args = parser['main'].parse_args(sys.argv[1:]) + + # Most subcommands have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) + + args.func(args) + +if __name__ == "__main__": + try: + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + emulator() + else: + commandline() + except KeyboardInterrupt: + sys.exit(0) From 02b9b71ffeb38e26af06b7cf06dc8e9e494f08f9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:11:39 +0200 Subject: [PATCH 020/107] remove exec functions from path Signed-off-by: Nico Schottelius --- module/cdist/path.py | 46 -------------------------------------------- 1 file changed, 46 deletions(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index 387481ca..964e29b8 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -91,52 +91,6 @@ class Path: """Use cat on the remote side for output""" self.run_or_fail(["cat", filename], remote=True) - def shell_run_or_debug_fail(self, script, *args, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - remote = true - - del kargs["remote"] - - log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - if remote: - remote_cat(script) - else: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - - exit_error("Command failed (shell): " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - - del kargs["remote"] - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - exit_error("Command failed: " + " ".join(*args)) - except OSError as error: - exit_error(" ".join(*args) + ": " + error.args[1]) - - def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) From 7bc4f74d50d0ff115e62981d17ed6b732590fa69 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:17:59 +0200 Subject: [PATCH 021/107] remove non path stuff from path Signed-off-by: Nico Schottelius --- module/cdist/path.py | 431 +------------------------------------------ 1 file changed, 4 insertions(+), 427 deletions(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index 964e29b8..c72d29a8 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -23,6 +23,9 @@ def file_to_list(filename): return lines +# FIXME: self.run_or_fail needs to be elsewhere! +# Exec? + class Path: """Class that handles path related configurations""" @@ -159,6 +162,7 @@ class Path: return object_paths + # FIXME def get_type_from_object(self, cdist_object): """Returns the first part (i.e. type) of an object""" return cdist_object.split(os.sep)[0] @@ -249,430 +253,3 @@ class Path: destination = os.path.join(self.bin_dir, type) log.debug("Linking %s to %s", source, destination) os.symlink(source, destination) - - def run_global_explores(self): - """Run global explorers""" - explorers = self.list_global_explorers() - if(len(explorers) == 0): - exit_error("No explorers found in", self.global_explorer_dir) - - self.transfer_global_explorers() - for explorer in explorers: - output = self.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.remote_global_explorer_path(explorer)) - - self.run_or_fail(cmd, stdout=output_fd, remote=True) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - # Based on bin/cdist-object-explorer-run - - # Transfering explorers for this type - type = self.get_type_from_object(cdist_object) - self.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.transfer_object_parameter(cdist_object) - - explorers = self.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - self.run_or_fail(remote_cmd, stdout=output_fd, remote=True) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.manifest_dir } - self.run_manifest(self.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.get_type_from_object(cdist_object) - manifest = self.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.object_dir(cdist_object), - "__object_id": self.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.base_dir, "core") - env['__cdist_local_base_dir'] = self.temp_dir - - # Submit information to new type emulator - env['__cdist_manifest'] = manifest - env['__cdist_type_base_dir'] = self.type_base_dir - - # Other environment stuff - if extra_env: - env.update(extra_env) - - self.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.object_dir(cdist_object), "require") - requirements = file_to_list(file) - type = self.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.out_dir - env["__object"] = self.object_dir(cdist_object) - env["__object_id"] = self.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.type_dir(type) - - if mode == "gencode": - paths = [ - self.type_dir(type, "gencode-local"), - self.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - self.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.object_dir(cdist_object) - remote_dir = self.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - self.run_or_fail([bin], remote=False) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.transfer_file(local_remote_code, remote_remote_code) - self.run_or_fail([remote_remote_code], remote=True) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = self.list_objects() - - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - - self.stage_prepare() - self.stage_run() - - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() - -def banner(args): - """Guess what :-)""" - print(BANNER) - sys.exit(0) - -def config(args): - """Configure remote system""" - process = {} - - time_start = datetime.datetime.now() - - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() - - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() - - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) - -def install(args): - """Install remote system""" - process = {} - -def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] - - parser = argparse.ArgumentParser(add_help=False) - - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) - - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) - - # Setup positional parameter, if not singleton - - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) - - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) - - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id - - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] - - # FIXME: verify object id - log.debug(args) - - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") - - object_source_file = os.path.join(object_dir, "source") - - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() - - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - exit_error(param_out_dir + ": " + error.args[1]) - - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) - - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) - else: - param_fd = open(file, "r") - param_old = param_fd.readlines() - param_fd.close() - - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() - - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() - - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() - - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) - - -def commandline(): - """Parse command line""" - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') - - # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + VERSION) - parser['main'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") - - # Banner - parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) - parser['banner'].set_defaults(func=banner) - - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) - - # Install - parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) - - for p in parser: - parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" - - args = parser['main'].parse_args(sys.argv[1:]) - - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) - - args.func(args) - -if __name__ == "__main__": - try: - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() - else: - commandline() - except KeyboardInterrupt: - sys.exit(0) From ceaf82f6df5494f96d31aab271a690e8bb5e13dc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 17:19:16 +0200 Subject: [PATCH 022/107] better comment Signed-off-by: Nico Schottelius --- module/cdist/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/cdist/path.py b/module/cdist/path.py index c72d29a8..e18f6582 100644 --- a/module/cdist/path.py +++ b/module/cdist/path.py @@ -1,4 +1,4 @@ -# Given paths from installation +# Hardcoded paths usually not changable REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") From 91022c3f7e7102833a4416cb45ae171f6e8d4b1e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 23 Sep 2011 17:19:45 +0200 Subject: [PATCH 023/107] handle ioerror if script does not exist Signed-off-by: Steven Armstrong --- bin/cdist | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/bin/cdist b/bin/cdist index fde7a251..d9f88b48 100755 --- a/bin/cdist +++ b/bin/cdist @@ -181,9 +181,12 @@ class Cdist: if remote: remote_cat(script) else: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() + try: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + except IOError as error: + raise CdistError(str(error)) raise CdistError("Command failed (shell): " + " ".join(*args)) except OSError as error: From 8e59f97800a0c9fecccbdb9517b1a8a581e5c933 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 18:55:23 +0200 Subject: [PATCH 024/107] begin to used cdist.path Signed-off-by: Nico Schottelius --- bin/cdist | 13 ++++++++----- {module => lib}/cdist/__init__.py | 0 {module => lib}/cdist/path.py | 0 3 files changed, 8 insertions(+), 5 deletions(-) rename {module => lib}/cdist/__init__.py (100%) rename {module => lib}/cdist/path.py (100%) diff --git a/bin/cdist b/bin/cdist index d9f88b48..69e0623f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,6 +20,10 @@ # # +# Add our own library path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib/python'))) + import argparse import datetime import logging @@ -32,6 +36,8 @@ import stat import sys import tempfile +import cdist.path + BANNER = """ .. . .x+=:. s dF @88> z` ^% :8 @@ -62,11 +68,6 @@ VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() -# Begin to split into parts -sys.path.insert(0, - os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib/python'))) - def file_to_list(filename): """Return list from \n seperated file""" if os.path.isfile(filename): @@ -94,6 +95,8 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] + self.path_info = cdist.path.Path(target_host, home) + # Setup directory paths self.temp_dir = tempfile.mkdtemp() diff --git a/module/cdist/__init__.py b/lib/cdist/__init__.py similarity index 100% rename from module/cdist/__init__.py rename to lib/cdist/__init__.py diff --git a/module/cdist/path.py b/lib/cdist/path.py similarity index 100% rename from module/cdist/path.py rename to lib/cdist/path.py From 2aa9f2ab8c8cb4d1349ea6ce98a9e3df8e5b0851 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 18:56:29 +0200 Subject: [PATCH 025/107] test: things to do Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 5439a1b9..8be1da54 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -9,6 +9,12 @@ CORE - allow cdist to run without $PATH setup: ./bin/cdist-deploy-to - support non-ssh access? +TESTS +----- +- multiple defines of object: + - fail if different parameters + - succeed if same parameters + USER INTERFACE -------------- - add support $__tmp? From 6139fab354b52ca56e5d4ffaea281d83b020a21e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 19:10:07 +0200 Subject: [PATCH 026/107] import cdist.path works Signed-off-by: Nico Schottelius --- bin/cdist | 10 +++++----- lib/cdist/path.py | 6 +++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 69e0623f..18e7845b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -20,10 +20,6 @@ # # -# Add our own library path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib/python'))) - import argparse import datetime import logging @@ -36,6 +32,10 @@ import stat import sys import tempfile +# Add our own library path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib'))) + import cdist.path BANNER = """ @@ -95,7 +95,7 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] - self.path_info = cdist.path.Path(target_host, home) + self.path_info = cdist.path.Path(target_host, base_dir=home) # Setup directory paths self.temp_dir = tempfile.mkdtemp() diff --git a/lib/cdist/path.py b/lib/cdist/path.py index e18f6582..5d72a23a 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -1,3 +1,7 @@ +import os +import sys +import tempfile + # Hardcoded paths usually not changable REMOTE_BASE_DIR = "/var/lib/cdist" REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") @@ -31,7 +35,7 @@ class Path: def __init__(self, target_host, base_dir=None): # Base and Temp Base - if home: + if base_dir: self.base_dir = base_dir else: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) From 26fad6f23ce1b656618eb64e1d367cf5757cf79f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 19:13:36 +0200 Subject: [PATCH 027/107] correct base dir, now relative to lib Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 5d72a23a..69bc5496 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -38,7 +38,7 @@ class Path: if base_dir: self.base_dir = base_dir else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) + self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) self.temp_dir = tempfile.mkdtemp() From 79173488ffd697ef89d8a387034eb3447cf3da61 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:08:55 +0200 Subject: [PATCH 028/107] cdist runs with path module loaded (unusued though) Signed-off-by: Nico Schottelius --- bin/cdist | 13 ++++++++----- lib/cdist/path.py | 31 ++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index 18e7845b..f6156825 100755 --- a/bin/cdist +++ b/bin/cdist @@ -32,11 +32,6 @@ import stat import sys import tempfile -# Add our own library path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib'))) - -import cdist.path BANNER = """ .. . .x+=:. s @@ -788,11 +783,19 @@ def commandline(): args.func(args) + if __name__ == "__main__": try: if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): emulator() else: + # Add our own library path + sys.path.insert(0, + os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib'))) + + import cdist.path + commandline() except KeyboardInterrupt: sys.exit(0) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 69bc5496..2050a1a3 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -1,3 +1,25 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging import os import sys import tempfile @@ -13,6 +35,10 @@ CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" TYPE_PREFIX = "__" +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +log = logging.getLogger() + + def file_to_list(filename): """Return list from \n seperated file""" if os.path.isfile(filename): @@ -33,7 +59,10 @@ def file_to_list(filename): class Path: """Class that handles path related configurations""" - def __init__(self, target_host, base_dir=None): + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + base_dir=None, debug=False): + # Base and Temp Base if base_dir: self.base_dir = base_dir From d4406cece3dc07e8a1f7a248b32c98c92f2895cb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:17:17 +0200 Subject: [PATCH 029/107] begin to replace first function with path module Signed-off-by: Nico Schottelius --- bin/cdist | 67 +++++++------------------------------------------------ 1 file changed, 8 insertions(+), 59 deletions(-) diff --git a/bin/cdist b/bin/cdist index f6156825..137f701f 100755 --- a/bin/cdist +++ b/bin/cdist @@ -90,72 +90,21 @@ class Cdist: self.target_host = target_host self.remote_prefix = ["ssh", "root@" + self.target_host] - self.path_info = cdist.path.Path(target_host, base_dir=home) - - # Setup directory paths - self.temp_dir = tempfile.mkdtemp() + self.path = cdist.path.Path(target_host, + initial_manifest=initial_manifest, + remote_user=remote_user, + base_dir=home, + debug=debug) self.debug = debug - if home: - self.base_dir = home - else: - self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)) - - self.conf_dir = os.path.join(self.base_dir, "conf") - self.cache_base_dir = os.path.join(self.base_dir, "cache") - self.cache_dir = os.path.join(self.cache_base_dir, self.target_host) - self.global_explorer_dir = os.path.join(self.conf_dir, "explorer") - self.lib_dir = os.path.join(self.base_dir, "lib") - self.manifest_dir = os.path.join(self.conf_dir, "manifest") - self.type_base_dir = os.path.join(self.conf_dir, "type") - - self.out_dir = os.path.join(self.temp_dir, "out") - os.mkdir(self.out_dir) - - self.global_explorer_out_dir = os.path.join(self.out_dir, "explorer") - os.mkdir(self.global_explorer_out_dir) - - self.object_base_dir = os.path.join(self.out_dir, "object") - - # Setup binary directory + contents - self.bin_dir = os.path.join(self.out_dir, "bin") - os.mkdir(self.bin_dir) - self.link_type_to_emulator() - - # List of type explorers transferred - self.type_explorers_transferred = {} - # objects self.objects_prepared = [] self.remote_user = remote_user - # Mostly static, but can be overwritten on user demand - if initial_manifest: - self.initial_manifest = initial_manifest - else: - self.initial_manifest = os.path.join(self.manifest_dir, "init") - def cleanup(self): - # Do not use in __del__: - # http://docs.python.org/reference/datamodel.html#customization - # "other globals referenced by the __del__() method may already have been deleted - # or in the process of being torn down (e.g. the import machinery shutting down)" - # - log.debug("Saving" + self.temp_dir + "to " + self.cache_dir) - # Remove previous cache - if os.path.exists(self.cache_dir): - shutil.rmtree(self.cache_dir) - shutil.move(self.temp_dir, self.cache_dir) - - def remote_mkdir(self, directory): - """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) - - def remote_cat(filename): - """Use cat on the remote side for output""" - self.run_or_fail(["cat", filename], remote=True) + self.path.cleanup() def shell_run_or_debug_fail(self, script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want @@ -415,8 +364,8 @@ class Cdist: """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") - self.remove_remote_dir(REMOTE_BASE_DIR) - self.remote_mkdir(REMOTE_BASE_DIR) + self.path.remove_remote_dir(REMOTE_BASE_DIR) + self.path.remote_mkdir(REMOTE_BASE_DIR) def run_initial_manifest(self): """Run the initial manifest""" From e14848ec1f88e1e79aade52fa4180d4a8d33db9e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:21:10 +0200 Subject: [PATCH 030/107] remove run* from cdist and put it into lib/exec Signed-off-by: Nico Schottelius --- bin/cdist | 49 --------------------------------- lib/cdist/exec.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+), 49 deletions(-) create mode 100644 lib/cdist/exec.py diff --git a/bin/cdist b/bin/cdist index 137f701f..30fec278 100755 --- a/bin/cdist +++ b/bin/cdist @@ -106,55 +106,6 @@ class Cdist: def cleanup(self): self.path.cleanup() - def shell_run_or_debug_fail(self, script, *args, **kargs): - # Manually execute /bin/sh, because sh -e does what we want - # and sh -c -e does not exit if /bin/false called - args[0][:0] = [ "/bin/sh", "-e" ] - - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - remote = true - - del kargs["remote"] - - log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - log.error("Code that raised the error:\n") - if remote: - remote_cat(script) - else: - try: - script_fd = open(script) - print(script_fd.read()) - script_fd.close() - except IOError as error: - raise CdistError(str(error)) - - raise CdistError("Command failed (shell): " + " ".join(*args)) - except OSError as error: - raise CdistError(" ".join(*args) + ": " + error.args[1]) - - def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = self.remote_prefix - - del kargs["remote"] - - log.debug("Exec: " + " ".join(*args)) - try: - subprocess.check_call(*args, **kargs) - except subprocess.CalledProcessError: - raise CdistError("Command failed: " + " ".join(*args)) - except OSError as error: - raise CdistError(" ".join(*args) + ": " + error.args[1]) - - def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py new file mode 100644 index 00000000..7a1421e0 --- /dev/null +++ b/lib/cdist/exec.py @@ -0,0 +1,70 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +def shell_run_or_debug_fail(self, script, *args, **kargs): + # Manually execute /bin/sh, because sh -e does what we want + # and sh -c -e does not exit if /bin/false called + args[0][:0] = [ "/bin/sh", "-e" ] + + remote = False + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + remote = true + + del kargs["remote"] + + log.debug("Shell exec cmd: %s", args) + log.debug("Shell exec env: %s", kargs['env']) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + log.error("Code that raised the error:\n") + if remote: + remote_cat(script) + else: + try: + script_fd = open(script) + print(script_fd.read()) + script_fd.close() + except IOError as error: + raise CdistError(str(error)) + + raise CdistError("Command failed (shell): " + " ".join(*args)) + except OSError as error: + raise CdistError(" ".join(*args) + ": " + error.args[1]) + + +def run_or_fail(self, *args, **kargs): + if "remote" in kargs: + if kargs["remote"]: + args[0][:0] = self.remote_prefix + + del kargs["remote"] + + log.debug("Exec: " + " ".join(*args)) + try: + subprocess.check_call(*args, **kargs) + except subprocess.CalledProcessError: + raise CdistError("Command failed: " + " ".join(*args)) + except OSError as error: + raise CdistError(" ".join(*args) + ": " + error.args[1]) + From 3925ba1c6e0bb7276a374836ec98f4f9e2c81552 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:35:13 +0200 Subject: [PATCH 031/107] pass remote_prefix to run_or_fail/shell Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 7a1421e0..a4cf4592 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -19,7 +19,8 @@ # # -def shell_run_or_debug_fail(self, script, *args, **kargs): + +def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] @@ -27,10 +28,11 @@ def shell_run_or_debug_fail(self, script, *args, **kargs): remote = False if "remote" in kargs: if kargs["remote"]: - args[0][:0] = self.remote_prefix + args[0][:0] = kargs["remote_prefix"] remote = true del kargs["remote"] + del kargs["remote_prefix"] log.debug("Shell exec cmd: %s", args) log.debug("Shell exec env: %s", kargs['env']) @@ -56,9 +58,10 @@ def shell_run_or_debug_fail(self, script, *args, **kargs): def run_or_fail(self, *args, **kargs): if "remote" in kargs: if kargs["remote"]: - args[0][:0] = self.remote_prefix + args[0][:0] = kargs["remote_prefix"] del kargs["remote"] + del kargs["remote_prefix"] log.debug("Exec: " + " ".join(*args)) try: @@ -67,4 +70,3 @@ def run_or_fail(self, *args, **kargs): raise CdistError("Command failed: " + " ".join(*args)) except OSError as error: raise CdistError(" ".join(*args) + ": " + error.args[1]) - From 8af45f83b20bd8a0e48f58eb5e7e9105bc76edda Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 23 Sep 2011 20:53:09 +0200 Subject: [PATCH 032/107] rearange remote_user Signed-off-by: Nico Schottelius --- bin/cdist | 17 ----------------- lib/cdist/path.py | 20 ++++++++++---------- 2 files changed, 10 insertions(+), 27 deletions(-) diff --git a/bin/cdist b/bin/cdist index 30fec278..67b96a09 100755 --- a/bin/cdist +++ b/bin/cdist @@ -88,7 +88,6 @@ class Cdist: initial_manifest=False, remote_user="root", home=None, debug=False): self.target_host = target_host - self.remote_prefix = ["ssh", "root@" + self.target_host] self.path = cdist.path.Path(target_host, initial_manifest=initial_manifest, @@ -98,7 +97,6 @@ class Cdist: self.debug = debug - # objects self.objects_prepared = [] self.remote_user = remote_user @@ -109,21 +107,6 @@ class Cdist: def remove_remote_dir(self, destination): self.run_or_fail(["rm", "-rf", destination], remote=True) - def transfer_dir(self, source, destination): - """Transfer directory and previously delete the remote destination""" - self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - - def transfer_file(self, source, destination): - """Transfer file""" - self.run_or_fail(["scp", "-q", source, - self.remote_user + "@" + - self.target_host + ":" + - destination]) - def global_explorer_output_path(self, explorer): """Returns path of the output for a global explorer""" return os.path.join(self.global_explorer_out_dir, explorer) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 2050a1a3..1e9d7195 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -38,6 +38,7 @@ TYPE_PREFIX = "__" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') log = logging.getLogger() +import cdist.exec def file_to_list(filename): """Return list from \n seperated file""" @@ -53,9 +54,6 @@ def file_to_list(filename): return lines -# FIXME: self.run_or_fail needs to be elsewhere! -# Exec? - class Path: """Class that handles path related configurations""" @@ -70,6 +68,10 @@ class Path: self.base_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)) self.temp_dir = tempfile.mkdtemp() + self.target_host = target_host + + self.remote_user = remote_user + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") @@ -98,8 +100,6 @@ class Path: # objects self.objects_prepared = [] - self.remote_user = remote_user - # Mostly static, but can be overwritten on user demand if initial_manifest: self.initial_manifest = initial_manifest @@ -121,26 +121,26 @@ class Path: def remote_mkdir(self, directory): """Create directory on remote side""" - self.run_or_fail(["mkdir", "-p", directory], remote=True) + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote=True) def remote_cat(filename): """Use cat on the remote side for output""" - self.run_or_fail(["cat", filename], remote=True) + cdist.exec.run_or_fail(["cat", filename], remote=True) def remove_remote_dir(self, destination): - self.run_or_fail(["rm", "-rf", destination], remote=True) + cdist.exec.run_or_fail(["rm", "-rf", destination], remote=True) def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" self.remove_remote_dir(destination) - self.run_or_fail(["scp", "-qr", source, + cdist.exec.run_or_fail(["scp", "-qr", source, self.remote_user + "@" + self.target_host + ":" + destination]) def transfer_file(self, source, destination): """Transfer file""" - self.run_or_fail(["scp", "-q", source, + cdist.exec.run_or_fail(["scp", "-q", source, self.remote_user + "@" + self.target_host + ":" + destination]) From 7b39169e3e55231eef729d83eea32be97ff49841 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Sep 2011 15:25:38 +0200 Subject: [PATCH 033/107] ignore all python cache dirs Signed-off-by: Nico Schottelius --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index d606aec7..fb9c495c 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,5 @@ doc/man/man*/docbook-xsl.css # Ignore cache for version control cache/ -# Python -bin/__pycache__/ +# Python / cache +__pycache__/ From a5bfd4119521b6758b7893945812ae2be2ad70dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Sep 2011 15:26:54 +0200 Subject: [PATCH 034/107] Detect owl Signed-off-by: Nico Schottelius --- conf/explorer/os | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/conf/explorer/os b/conf/explorer/os index e922c067..268dae24 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -65,6 +65,11 @@ if [ -f /etc/SuSE-release ]; then exit 0 fi +if uname -r | grep -s '.owl' >/dev/null 2>&1; then + echo owl + exit 0 +fi + if [ -f /etc/cdist-preos ]; then echo preos exit 0 From c10fedaf8c9f87c36e0a0bb2602de8a6ff46edec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Sep 2011 21:10:18 +0200 Subject: [PATCH 035/107] it's not preos, but cdist-preos Signed-off-by: Nico Schottelius --- conf/explorer/os | 2 +- doc/changelog | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/explorer/os b/conf/explorer/os index 268dae24..3e1582ec 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -71,7 +71,7 @@ if uname -r | grep -s '.owl' >/dev/null 2>&1; then fi if [ -f /etc/cdist-preos ]; then - echo preos + echo cdist-preos exit 0 fi diff --git a/doc/changelog b/doc/changelog index b1149eb1..a08efb34 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,3 +1,6 @@ +2.0.2: + * Add support for detection of OpenWall Linux (Matthias Teege) + 2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors * Bugfix core: Various smaller bugs in string concatenation From 98f4ec9f3e6334768e3f7065dfd66ef9e13c772c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 09:24:49 +0200 Subject: [PATCH 036/107] document asciidoc requirement Signed-off-by: Nico Schottelius --- README | 1 + 1 file changed, 1 insertion(+) diff --git a/README b/README index 5e60a093..b8181757 100644 --- a/README +++ b/README @@ -78,6 +78,7 @@ cdist was tested or is know to run on at least * A posix like shell * Python (>= 3.2 required) * SSH-Client + * Asciidoc (for building the manpages) ### Client ("target host") From fd9fb13606aa54687d36f9b70965cb6a9e086829 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 09:26:58 +0200 Subject: [PATCH 037/107] and remove asciidoc/gmake dep from the comment Signed-off-by: Nico Schottelius --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index b8181757..daa12f70 100644 --- a/README +++ b/README @@ -99,7 +99,7 @@ To install cdist, execute the following commands: cd cdist export PATH=$PATH:$(pwd -P)/bin - # If you want the manpages (requires gmake and asciidoc to be installed) + # If you want the manpages ./build.sh man export MANPATH=$MANPATH:$(pwd -P)/doc/man From bc9bc37aab80bc85d991621da9addd016ac9d0d4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:17:02 +0200 Subject: [PATCH 038/107] use remote_prefix internally Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 26 +++++++++++++------------- lib/cdist/path.py | 6 +++--- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index a4cf4592..aca1d689 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -19,19 +19,21 @@ # # +import logging +import subprocess + +logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') +log = logging.getLogger() + def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] - remote = False - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = kargs["remote_prefix"] - remote = true - - del kargs["remote"] + if "remote_prefix" in kargs: + remote = True + args[0][:0] = kargs["remote_prefix"] del kargs["remote_prefix"] log.debug("Shell exec cmd: %s", args) @@ -41,6 +43,7 @@ def shell_run_or_debug_fail(script, *args, **kargs): except subprocess.CalledProcessError: log.error("Code that raised the error:\n") if remote: + # FIXME: included in Path! remote_cat(script) else: try: @@ -55,12 +58,9 @@ def shell_run_or_debug_fail(script, *args, **kargs): raise CdistError(" ".join(*args) + ": " + error.args[1]) -def run_or_fail(self, *args, **kargs): - if "remote" in kargs: - if kargs["remote"]: - args[0][:0] = kargs["remote_prefix"] - - del kargs["remote"] +def run_or_fail(*args, **kargs): + if "remote_prefix" in kargs: + args[0][:0] = kargs["remote_prefix"] del kargs["remote_prefix"] log.debug("Exec: " + " ".join(*args)) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 1e9d7195..c2cef1a6 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -121,14 +121,14 @@ class Path: def remote_mkdir(self, directory): """Create directory on remote side""" - cdist.exec.run_or_fail(["mkdir", "-p", directory], remote=True) + cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) def remote_cat(filename): """Use cat on the remote side for output""" - cdist.exec.run_or_fail(["cat", filename], remote=True) + cdist.exec.run_or_fail(["cat", filename], remote_prefix=self.remote_prefix) def remove_remote_dir(self, destination): - cdist.exec.run_or_fail(["rm", "-rf", destination], remote=True) + cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) def transfer_dir(self, source, destination): """Transfer directory and previously delete the remote destination""" From 8f2e5bb8c869bd0cce84ae13ccf84756c70298cb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:25:51 +0200 Subject: [PATCH 039/107] cleanup and move error class to init Signed-off-by: Nico Schottelius --- bin/cdist | 150 +----------------------------------------- lib/cdist/__init__.py | 24 +++++++ 2 files changed, 25 insertions(+), 149 deletions(-) diff --git a/bin/cdist b/bin/cdist index 67b96a09..369fbd29 100755 --- a/bin/cdist +++ b/bin/cdist @@ -77,10 +77,6 @@ def file_to_list(filename): return lines -class CdistError(Exception): - """Base exception class for this project""" - pass - class Cdist: """Cdist main class to hold arbitrary data""" @@ -104,153 +100,9 @@ class Cdist: def cleanup(self): self.path.cleanup() - def remove_remote_dir(self, destination): - self.run_or_fail(["rm", "-rf", destination], remote=True) - - def global_explorer_output_path(self, explorer): - """Returns path of the output for a global explorer""" - return os.path.join(self.global_explorer_out_dir, explorer) - - def type_explorer_output_dir(self, cdist_object): - """Returns and creates dir of the output for a type explorer""" - dir = os.path.join(self.object_dir(cdist_object), "explorer") - if not os.path.isdir(dir): - os.mkdir(dir) - - return dir - - def remote_global_explorer_path(self, explorer): - """Returns path to the remote explorer""" - return os.path.join(REMOTE_GLOBAL_EXPLORER_DIR, explorer) - - def list_global_explorers(self): - """Return list of available explorers""" - return os.listdir(self.global_explorer_dir) - - def list_type_explorers(self, type): - """Return list of available explorers for a specific type""" - dir = self.type_dir(type, "explorer") - if os.path.isdir(dir): - list = os.listdir(dir) - else: - list = [] - - log.debug("Explorers for %s in %s: %s", type, dir, list) - - return list - - def list_types(self): - return os.listdir(self.type_base_dir) - - def list_object_paths(self, starting_point): - """Return list of paths of existing objects""" - object_paths = [] - - for content in os.listdir(starting_point): - full_path = os.path.join(starting_point, content) - if os.path.isdir(full_path): - object_paths.extend(self.list_object_paths(starting_point = full_path)) - - # Directory contains .cdist -> is an object - if content == DOT_CDIST: - object_paths.append(starting_point) - - return object_paths - - def get_type_from_object(self, cdist_object): - """Returns the first part (i.e. type) of an object""" - return cdist_object.split(os.sep)[0] - - def get_object_id_from_object(self, cdist_object): - """Returns everything but the first part (i.e. object_id) of an object""" - return os.sep.join(cdist_object.split(os.sep)[1:]) - - def object_dir(self, cdist_object): - """Returns the full path to the object (including .cdist)""" - return os.path.join(self.object_base_dir, cdist_object, DOT_CDIST) - - def remote_object_dir(self, cdist_object): - """Returns the remote full path to the object (including .cdist)""" - return os.path.join(REMOTE_OBJECT_DIR, cdist_object, DOT_CDIST) - - def object_parameter_dir(self, cdist_object): - """Returns the dir to the object parameter""" - return os.path.join(self.object_dir(cdist_object), "parameter") - - def remote_object_parameter_dir(self, cdist_object): - """Returns the remote dir to the object parameter""" - return os.path.join(self.remote_object_dir(cdist_object), "parameter") - - def object_code_paths(self, cdist_object): - """Return paths to code scripts of object""" - return [os.path.join(self.object_dir(cdist_object), "code-local"), - os.path.join(self.object_dir(cdist_object), "code-remote")] - - def list_objects(self): - """Return list of existing objects""" - - objects = [] - if os.path.isdir(self.object_base_dir): - object_paths = self.list_object_paths(self.object_base_dir) - - for path in object_paths: - objects.append(os.path.relpath(path, self.object_base_dir)) - - return objects - - def type_dir(self, type, *args): - """Return directory the type""" - return os.path.join(self.type_base_dir, type, *args) - - def remote_type_explorer_dir(self, type): - """Return remote directory that holds the explorers of a type""" - return os.path.join(REMOTE_TYPE_DIR, type, "explorer") - - def transfer_object_parameter(self, cdist_object): - """Transfer the object parameter to the remote destination""" - # Create base path before using mkdir -p - self.remote_mkdir(self.remote_object_parameter_dir(cdist_object)) - - # Synchronise parameter dir afterwards - self.transfer_dir(self.object_parameter_dir(cdist_object), - self.remote_object_parameter_dir(cdist_object)) - - def transfer_global_explorers(self): - """Transfer the global explorers""" - self.remote_mkdir(REMOTE_GLOBAL_EXPLORER_DIR) - self.transfer_dir(self.global_explorer_dir, REMOTE_GLOBAL_EXPLORER_DIR) - - def transfer_type_explorers(self, type): - """Transfer explorers of a type, but only once""" - if type in self.type_explorers_transferred: - log.debug("Skipping retransfer for explorers of %s", type) - return - else: - # Do not retransfer - self.type_explorers_transferred[type] = 1 - - src = self.type_dir(type, "explorer") - remote_base = os.path.join(REMOTE_TYPE_DIR, type) - dst = self.remote_type_explorer_dir(type) - - # Only continue, if there is at least the directory - if os.path.isdir(src): - # Ensure that the path exists - self.remote_mkdir(remote_base) - self.transfer_dir(src, dst) - - - def link_type_to_emulator(self): - """Link type names to cdist-type-emulator""" - source = os.path.abspath(sys.argv[0]) - for type in self.list_types(): - destination = os.path.join(self.bin_dir, type) - log.debug("Linking %s to %s", source, destination) - os.symlink(source, destination) - def run_global_explores(self): """Run global explorers""" - explorers = self.list_global_explorers() + explorers = self.path.list_global_explorers() if(len(explorers) == 0): raise CdistError("No explorers found in", self.global_explorer_dir) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index e69de29b..a1afebbd 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +class Error(Exception): + """Base exception class for this project""" + pass From e6a903fd967be6cde4e7b6585aceaa42954f4049 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:28:57 +0200 Subject: [PATCH 040/107] cleanup logger stuff, remove more path stuff from cdist bin Signed-off-by: Nico Schottelius --- bin/cdist | 10 ---------- lib/cdist/exec.py | 4 +--- lib/cdist/path.py | 3 +-- 3 files changed, 2 insertions(+), 15 deletions(-) diff --git a/bin/cdist b/bin/cdist index 369fbd29..ed71249c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -48,16 +48,6 @@ BANNER = """ "P' "" "" """ -# Given paths from installation -REMOTE_BASE_DIR = "/var/lib/cdist" -REMOTE_CONF_DIR = os.path.join(REMOTE_BASE_DIR, "conf") -REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") -REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") -REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") - -CODE_HEADER = "#!/bin/sh -e\n" -DOT_CDIST = ".cdist" -TYPE_PREFIX = "__" VERSION = "2.0.1" logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index aca1d689..4b61a097 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -22,9 +22,7 @@ import logging import subprocess -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -log = logging.getLogger() - +log = logging.getLogger(__name__) def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want diff --git a/lib/cdist/path.py b/lib/cdist/path.py index c2cef1a6..401e3c46 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -35,8 +35,7 @@ CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" TYPE_PREFIX = "__" -logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') -log = logging.getLogger() +log = logging.getLogger(__name__) import cdist.exec From cf920ca3e96d302e00b6827968ff1a387c574bed Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 10:53:34 +0200 Subject: [PATCH 041/107] prefix issues Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 91dbc72f..4e07dd96 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -1,16 +1,3 @@ -2.0.1: - -- Rewrite cdist-type-emulator - - Remove legacy code in cdist - - Remove cdist-config - - Remove man1/cdist-type-emulator.text - - Remove the PATH=... part from the README - - - how to access output dir? - - Test: - __cdist_type_base_dir=$(pwd -P)/conf/type __file - - Fix / rewrite cdist-quickstart - write tutorial!!!!!!!!! @@ -49,3 +36,7 @@ http://www.youtube.com/watch?v=PRMjzy48eTI - Setup __debug, if -d is given, so other tools can reuse it + (-> non core feature! + +- remote_prefix: + scp vs. ssh issue From 16d58dcac73f71352d5b77bbf5df2d78988bd68e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:05:18 +0200 Subject: [PATCH 042/107] move out banner, fail at emulator Signed-off-by: Nico Schottelius --- bin/cdist | 204 ++++++++++++++++++------------------------ bin/cdist.py | 1 - lib/cdist/__init__.py | 3 + lib/cdist/banner.py | 46 ++++++++++ lib/cdist/path.py | 8 +- 5 files changed, 140 insertions(+), 122 deletions(-) delete mode 120000 bin/cdist.py create mode 100644 lib/cdist/banner.py diff --git a/bin/cdist b/bin/cdist index ed71249c..15c3ff62 100755 --- a/bin/cdist +++ b/bin/cdist @@ -32,40 +32,7 @@ import stat import sys import tempfile - -BANNER = """ - .. . .x+=:. s - dF @88> z` ^% :8 - '88bu. %8P . . +# +# + +import logging +import sys + +log = logging.getLogger(__name__) + +BANNER = """ + .. . .x+=:. s + dF @88> z` ^% :8 + '88bu. %8P . Date: Mon, 26 Sep 2011 11:07:45 +0200 Subject: [PATCH 043/107] move TYPE_PREFIX back into main, as it's only needed there and should work without module Signed-off-by: Nico Schottelius --- bin/cdist | 4 +++- lib/cdist/__init__.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 15c3ff62..f4d67616 100755 --- a/bin/cdist +++ b/bin/cdist @@ -34,6 +34,8 @@ import tempfile log = logging.getLogger(__name__) +TYPE_PREFIX = "__" + class Cdist: """Cdist main class to hold arbitrary data""" @@ -482,7 +484,7 @@ if __name__ == "__main__": '../lib'))) import cdist - if re.match(cdist.TYPE_PREFIX, os.path.basename(sys.argv[0])): + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): emulator() else: import cdist.banner diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 6bcc93d2..192e5001 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -19,7 +19,6 @@ # # -TYPE_PREFIX = "__" VERSION = "2.0.2" class Error(Exception): From 4eec4d96270464d8b5275a71e9d40e6f44f54bb4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:09:58 +0200 Subject: [PATCH 044/107] only setup library path in main, not emulator Signed-off-by: Nico Schottelius --- bin/cdist | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index f4d67616..d6a6da3b 100755 --- a/bin/cdist +++ b/bin/cdist @@ -478,15 +478,14 @@ if __name__ == "__main__": try: logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - # Import generic cdist options - sys.path.insert(0, - os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib'))) - import cdist - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): emulator() else: + cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib')) + sys.path.insert(0, cdist_lib) + + import cdist import cdist.banner import cdist.exec import cdist.path From 6d75016139cec7d14a9d937eecabd3c32f4bd885 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:18:36 +0200 Subject: [PATCH 045/107] make type emulator load again Signed-off-by: Nico Schottelius --- bin/cdist | 120 ++++---------------------------------- lib/cdist/emulator.py | 132 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 108 deletions(-) create mode 100755 lib/cdist/emulator.py diff --git a/bin/cdist b/bin/cdist index d6a6da3b..3b19e388 100755 --- a/bin/cdist +++ b/bin/cdist @@ -146,9 +146,16 @@ class Cdist: env['__cdist_local_base_dir'] = self.path.temp_dir # Submit information to new type emulator + + # Required for recording source env['__cdist_manifest'] = manifest + + # Required to find types env['__cdist_type_base_dir'] = self.path.type_base_dir + # Required for loading emulator :-) + env['__cdist_python_lib'] = cdist_lib + # Other environment stuff if extra_env: env.update(extra_env) @@ -306,112 +313,6 @@ def install(args): """Install remote system""" process = {} -def emulator(): - """Emulate type commands (i.e. __file and co)""" - type = os.path.basename(sys.argv[0]) - type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) - param_dir = os.path.join(type_dir, "parameter") - global_dir = os.environ['__global'] - object_source = os.environ['__cdist_manifest'] - - parser = argparse.ArgumentParser(add_help=False) - - # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=False) - - # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): - argument = "--" + parameter - parser.add_argument(argument, action='store', required=True) - - # Setup positional parameter, if not singleton - - if not os.path.isfile(os.path.join(type_dir, "singleton")): - parser.add_argument("object_id", nargs=1) - - # And finally verify parameter - args = parser.parse_args(sys.argv[1:]) - - # Setup object_id - if os.path.isfile(os.path.join(type_dir, "singleton")): - object_id = "singleton" - else: - object_id = args.object_id[0] - del args.object_id - - # FIXME: / hardcoded - better portable solution available? - if object_id[0] == '/': - object_id = object_id[1:] - - # FIXME: verify object id - log.debug(args) - - object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) - param_out_dir = os.path.join(object_dir, "parameter") - - object_source_file = os.path.join(object_dir, "source") - - if os.path.exists(param_out_dir): - object_exists = True - old_object_source_fd = open(object_source_file, "r") - old_object_source = old_object_source_fd.readlines() - old_object_source_fd.close() - - else: - object_exists = False - try: - os.makedirs(param_out_dir, exist_ok=True) - except OSError as error: - raise CdistError(param_out_dir + ": " + error.args[1]) - - # Record parameter - params = vars(args) - for param in params: - value = getattr(args, param) - if value: - file = os.path.join(param_out_dir, param) - log.debug(file + "<-" + param + " = " + value) - - # Already exists, verify all parameter are the same - if object_exists: - if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) - sys.exit(1) - else: - param_fd = open(file, "r") - param_old = param_fd.readlines() - param_fd.close() - - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) - else: - param_fd = open(file, "w") - param_fd.writelines(value) - param_fd.close() - - # Record requirements - if "__require" in os.environ: - requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) - require_fd = open(os.path.join(object_dir, "require"), "a") - require_fd.writelines(requirements.split(" ")) - require_fd.close() - - # Record / Append source - source_fd = open(os.path.join(object_dir, "source"), "a") - source_fd.writelines(object_source) - source_fd.close() - - # sys.exit(1) - print("Finished " + type + "/" + object_id + repr(params)) - - def commandline(): """Parse command line""" # Construct parser others can reuse @@ -477,9 +378,12 @@ def commandline(): if __name__ == "__main__": try: logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - emulator() + cdist_lib = os.environ["__cdist_python_lib"] + sys.path.insert(0, cdist_lib) + import cdist.emulator + cdist.emulator.emulator(sys.argv) else: cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), '../lib')) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py new file mode 100755 index 00000000..0e693dbe --- /dev/null +++ b/lib/cdist/emulator.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import argparse +import logging +import os +import sys + +log = logging.getLogger(__name__) + +def emulator(argv): + """Emulate type commands (i.e. __file and co)""" + type = os.path.basename(argv[0]) + type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) + param_dir = os.path.join(type_dir, "parameter") + global_dir = os.environ['__global'] + object_source = os.environ['__cdist_manifest'] + + parser = argparse.ArgumentParser(add_help=False) + + # Setup optional parameters + for parameter in file_to_list(os.path.join(param_dir, "optional")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=False) + + # Setup required parameters + for parameter in file_to_list(os.path.join(param_dir, "required")): + argument = "--" + parameter + parser.add_argument(argument, action='store', required=True) + + # Setup positional parameter, if not singleton + + if not os.path.isfile(os.path.join(type_dir, "singleton")): + parser.add_argument("object_id", nargs=1) + + # And finally verify parameter + args = parser.parse_args(argv[1:]) + + # Setup object_id + if os.path.isfile(os.path.join(type_dir, "singleton")): + object_id = "singleton" + else: + object_id = args.object_id[0] + del args.object_id + + # FIXME: / hardcoded - better portable solution available? + if object_id[0] == '/': + object_id = object_id[1:] + + # FIXME: verify object id + log.debug(args) + + object_dir = os.path.join(global_dir, "object", type, + object_id, DOT_CDIST) + param_out_dir = os.path.join(object_dir, "parameter") + + object_source_file = os.path.join(object_dir, "source") + + if os.path.exists(param_out_dir): + object_exists = True + old_object_source_fd = open(object_source_file, "r") + old_object_source = old_object_source_fd.readlines() + old_object_source_fd.close() + + else: + object_exists = False + try: + os.makedirs(param_out_dir, exist_ok=True) + except OSError as error: + raise CdistError(param_out_dir + ": " + error.args[1]) + + # Record parameter + params = vars(args) + for param in params: + value = getattr(args, param) + if value: + file = os.path.join(param_out_dir, param) + log.debug(file + "<-" + param + " = " + value) + + # Already exists, verify all parameter are the same + if object_exists: + if not os.path.isfile(file): + print("New parameter + " + param + "specified, aborting") + print("Source = " + old_object_source + "new =" + object_source) + sys.exit(1) + else: + param_fd = open(file, "r") + param_old = param_fd.readlines() + param_fd.close() + + if(param_old != param): + print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + print("Sources: " + " ".join(old_object_source) + " and " + object_source) + sys.exit(1) + else: + param_fd = open(file, "w") + param_fd.writelines(value) + param_fd.close() + + # Record requirements + if "__require" in os.environ: + requirements = os.environ['__require'] + print(object_id + ":Writing requirements: " + requirements) + require_fd = open(os.path.join(object_dir, "require"), "a") + require_fd.writelines(requirements.split(" ")) + require_fd.close() + + # Record / Append source + source_fd = open(os.path.join(object_dir, "source"), "a") + source_fd.writelines(object_source) + source_fd.close() + + # sys.exit(1) + print("Finished " + type + "/" + object_id + repr(params)) From 74dc5b96cb3af00f8fb6ac3dd805f69de40d2737 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:21:04 +0200 Subject: [PATCH 046/107] type emulator begins to run Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 8 +++++--- lib/cdist/exec.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 0e693dbe..ecdfff25 100755 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -24,6 +24,8 @@ import logging import os import sys +import cdist.path + log = logging.getLogger(__name__) def emulator(argv): @@ -37,12 +39,12 @@ def emulator(argv): parser = argparse.ArgumentParser(add_help=False) # Setup optional parameters - for parameter in file_to_list(os.path.join(param_dir, "optional")): + for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): argument = "--" + parameter parser.add_argument(argument, action='store', required=False) # Setup required parameters - for parameter in file_to_list(os.path.join(param_dir, "required")): + for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): argument = "--" + parameter parser.add_argument(argument, action='store', required=True) @@ -69,7 +71,7 @@ def emulator(argv): log.debug(args) object_dir = os.path.join(global_dir, "object", type, - object_id, DOT_CDIST) + object_id, cdist.path.DOT_CDIST) param_out_dir = os.path.join(object_dir, "parameter") object_source_file = os.path.join(object_dir, "source") diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 4b61a097..09e4e8a4 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -29,6 +29,7 @@ def shell_run_or_debug_fail(script, *args, **kargs): # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] + remote = False if "remote_prefix" in kargs: remote = True args[0][:0] = kargs["remote_prefix"] From 0197f8da13ffdff8103a1a35c118f82ffca6968e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:25:22 +0200 Subject: [PATCH 047/107] more cleanups from refactoring Signed-off-by: Nico Schottelius --- bin/cdist | 5 +++-- lib/cdist/path.py | 17 +++++++++-------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/bin/cdist b/bin/cdist index 3b19e388..f511d78e 100755 --- a/bin/cdist +++ b/bin/cdist @@ -35,6 +35,7 @@ import tempfile log = logging.getLogger(__name__) TYPE_PREFIX = "__" +CODE_HEADER = "#!/bin/sh -e\n" class Cdist: """Cdist main class to hold arbitrary data""" @@ -102,7 +103,7 @@ class Cdist: log.debug("%s exploring %s using %s storing to %s", cdist_object, explorer, remote_cmd, output) - self.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() def init_deploy(self): @@ -240,7 +241,7 @@ class Cdist: self.run_initial_manifest() old_objects = [] - objects = self.list_objects() + objects = self.path.list_objects() # Continue process until no new objects are created anymore while old_objects != objects: diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 16aa51ea..0fa753a8 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -21,6 +21,7 @@ import logging import os +import shutil import sys import tempfile @@ -31,7 +32,6 @@ REMOTE_OBJECT_DIR = os.path.join(REMOTE_BASE_DIR, "object") REMOTE_TYPE_DIR = os.path.join(REMOTE_CONF_DIR, "type") REMOTE_GLOBAL_EXPLORER_DIR = os.path.join(REMOTE_CONF_DIR, "explorer") -CODE_HEADER = "#!/bin/sh -e\n" DOT_CDIST = ".cdist" log = logging.getLogger(__name__) @@ -55,9 +55,13 @@ def file_to_list(filename): class Path: """Class that handles path related configurations""" - def __init__(self, target_host, - initial_manifest=False, remote_user="root", - remote_prefix=False, base_dir=None, debug=False): + def __init__(self, + target_host, + remote_user, + remote_prefix, + initial_manifest=False, + base_dir=None, + debug=False): # Base and Temp Base if base_dir: @@ -69,10 +73,7 @@ class Path: self.target_host = target_host self.remote_user = remote_user - if remote_prefix: - self.remote_prefix = remote_prefix - else: - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + self.remote_prefix = remote_prefix self.conf_dir = os.path.join(self.base_dir, "conf") self.cache_base_dir = os.path.join(self.base_dir, "cache") From 35e33570d18eadd88d929a3b213a504218152a72 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:45:19 +0200 Subject: [PATCH 048/107] also move out config Signed-off-by: Nico Schottelius --- bin/cdist | 286 +------------------------------- lib/cdist/config.py | 393 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 399 insertions(+), 280 deletions(-) create mode 100755 lib/cdist/config.py diff --git a/bin/cdist b/bin/cdist index f511d78e..4dd4f971 100755 --- a/bin/cdist +++ b/bin/cdist @@ -33,283 +33,14 @@ import sys import tempfile log = logging.getLogger(__name__) +real_me = os.path.dirname(os.path.realpath(__file__)) +cdist_lib = os.path.abspath(os.path.join(real_me, '../lib')) + +sys.path.insert(0, cdist_lib) TYPE_PREFIX = "__" CODE_HEADER = "#!/bin/sh -e\n" -class Cdist: - """Cdist main class to hold arbitrary data""" - - def __init__(self, target_host, - initial_manifest=False, remote_user="root", - home=None, debug=False): - - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user - self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] - - self.path = cdist.path.Path(self.target_host, - initial_manifest=initial_manifest, - remote_user=self.remote_user, - remote_prefix=self.remote_prefix, - base_dir=home, - debug=debug) - - - self.objects_prepared = [] - - def cleanup(self): - self.path.cleanup() - - def run_global_explores(self): - """Run global explorers""" - explorers = self.path.list_global_explorers() - if(len(explorers) == 0): - raise CdistError("No explorers found in", self.path.global_explorer_dir) - - self.path.transfer_global_explorers() - for explorer in explorers: - output = self.path.global_explorer_output_path(explorer) - output_fd = open(output, mode='w') - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append(self.path.remote_global_explorer_path(explorer)) - - cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() - - def run_type_explorer(self, cdist_object): - """Run type specific explorers for objects""" - - type = self.path.get_type_from_object(cdist_object) - self.path.transfer_type_explorers(type) - - cmd = [] - cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) - cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) - cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) - cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) - cmd.append("__object_fq=" + cdist_object) - - # Need to transfer at least the parameters for objects to be useful - self.path.transfer_object_parameter(cdist_object) - - explorers = self.path.list_type_explorers(type) - for explorer in explorers: - remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] - output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) - output_fd = open(output, mode='w') - log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) - - cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) - output_fd.close() - - def init_deploy(self): - """Ensure the base directories are cleaned up""" - log.debug("Creating clean directory structure") - - self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) - self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - - def run_initial_manifest(self): - """Run the initial manifest""" - env = { "__manifest" : self.path.manifest_dir } - self.run_manifest(self.path.initial_manifest, extra_env=env) - - def run_type_manifest(self, cdist_object): - """Run manifest for a specific object""" - type = self.path.get_type_from_object(cdist_object) - manifest = self.path.type_dir(type, "manifest") - - log.debug("%s: Running %s", cdist_object, manifest) - if os.path.exists(manifest): - env = { "__object" : self.path.object_dir(cdist_object), - "__object_id": self.path.get_object_id_from_object(cdist_object), - "__object_fq": cdist_object, - "__type": self.path.type_dir(type) - } - self.run_manifest(manifest, extra_env=env) - - def run_manifest(self, manifest, extra_env=None): - """Run a manifest""" - log.debug("Running manifest %s, env=%s", manifest, extra_env) - env = os.environ.copy() - env['PATH'] = self.path.bin_dir + ":" + env['PATH'] - - # Information required in every manifest - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.path.base_dir, "core") - env['__cdist_local_base_dir'] = self.path.temp_dir - - # Submit information to new type emulator - - # Required for recording source - env['__cdist_manifest'] = manifest - - # Required to find types - env['__cdist_type_base_dir'] = self.path.type_base_dir - - # Required for loading emulator :-) - env['__cdist_python_lib'] = cdist_lib - - # Other environment stuff - if extra_env: - env.update(extra_env) - - cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) - - def object_run(self, cdist_object, mode): - """Run gencode or code for an object""" - log.debug("Running %s from %s", mode, cdist_object) - file=os.path.join(self.path.object_dir(cdist_object), "require") - requirements = cdist.path.file_to_list(file) - type = self.path.get_type_from_object(cdist_object) - - for requirement in requirements: - log.debug("Object %s requires %s", cdist_object, requirement) - self.object_run(requirement, mode=mode) - - # - # Setup env Variable: - # - env = os.environ.copy() - env['__target_host'] = self.target_host - env['__global'] = self.path.out_dir - env["__object"] = self.path.object_dir(cdist_object) - env["__object_id"] = self.path.get_object_id_from_object(cdist_object) - env["__object_fq"] = cdist_object - env["__type"] = self.path.type_dir(type) - - if mode == "gencode": - paths = [ - self.path.type_dir(type, "gencode-local"), - self.path.type_dir(type, "gencode-remote") - ] - for bin in paths: - if os.path.isfile(bin): - # omit "gen" from gencode and use it for output base - outfile=os.path.join(self.path.object_dir(cdist_object), - os.path.basename(bin)[3:]) - - outfile_fd = open(outfile, "w") - - # Need to flush to ensure our write is done before stdout write - outfile_fd.write(CODE_HEADER) - outfile_fd.flush() - - cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) - outfile_fd.close() - - status = os.stat(outfile) - - # Remove output if empty, else make it executable - if status.st_size == len(CODE_HEADER): - os.unlink(outfile) - else: - # Add header and make executable - identically to 0o700 - os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) - - # Mark object as changed - open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() - - - if mode == "code": - local_dir = self.path.object_dir(cdist_object) - remote_dir = self.path.remote_object_dir(cdist_object) - - bin = os.path.join(local_dir, "code-local") - if os.path.isfile(bin): - cdist.exec.run_or_fail([bin]) - - - local_remote_code = os.path.join(local_dir, "code-remote") - remote_remote_code = os.path.join(remote_dir, "code-remote") - if os.path.isfile(local_remote_code): - self.path.transfer_file(local_remote_code, remote_remote_code) - # FIXME: remote_prefix - cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) - - def stage_prepare(self): - """Do everything for a deploy, minus the actual code stage""" - self.init_deploy() - self.run_global_explores() - self.run_initial_manifest() - - old_objects = [] - objects = self.path.list_objects() - - # Continue process until no new objects are created anymore - while old_objects != objects: - log.debug("Prepare stage") - old_objects = list(objects) - for cdist_object in objects: - if cdist_object in self.objects_prepared: - log.debug("Skipping rerun of object %s", cdist_object) - continue - else: - self.run_type_explorer(cdist_object) - self.run_type_manifest(cdist_object) - self.objects_prepared.append(cdist_object) - - objects = self.path.list_objects() - - def stage_run(self): - """The final (and real) step of deployment""" - log.debug("Actual run objects") - # Now do the final steps over the existing objects - for cdist_object in self.path.list_objects(): - log.debug("Run object: %s", cdist_object) - self.object_run(cdist_object, mode="gencode") - self.object_run(cdist_object, mode="code") - - def deploy_to(self): - """Mimic the old deploy to: Deploy to one host""" - log.info("Deploying to " + self.target_host) - time_start = datetime.datetime.now() - - self.stage_prepare() - self.stage_run() - - time_end = datetime.datetime.now() - duration = time_end - time_start - log.info("Finished run of %s in %s seconds", - self.target_host, - duration.total_seconds()) - - def deploy_and_cleanup(self): - """Do what is most often done: deploy & cleanup""" - self.deploy_to() - self.cleanup() - -def config(args): - """Configure remote system""" - process = {} - - time_start = datetime.datetime.now() - - for host in args.host: - c = Cdist(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) - if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) - process[host].start() - else: - c.deploy_and_cleanup() - - if args.parallel: - for p in process.keys(): - log.debug("Joining %s", p) - process[p].join() - - time_end = datetime.datetime.now() - log.info("Total processing time for %s host(s): %s", len(args.host), - (time_end - time_start).total_seconds()) - def install(args): """Install remote system""" process = {} @@ -355,7 +86,7 @@ def commandline(): # Config parser['config'] = parser['sub'].add_parser('config', parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) + parser['config'].set_defaults(func=cdist.config.config) # Install parser['install'] = parser['sub'].add_parser('install', @@ -381,17 +112,12 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - cdist_lib = os.environ["__cdist_python_lib"] - sys.path.insert(0, cdist_lib) import cdist.emulator cdist.emulator.emulator(sys.argv) else: - cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib')) - sys.path.insert(0, cdist_lib) - import cdist import cdist.banner + import cdist.config import cdist.exec import cdist.path diff --git a/lib/cdist/config.py b/lib/cdist/config.py new file mode 100755 index 00000000..a5a2252f --- /dev/null +++ b/lib/cdist/config.py @@ -0,0 +1,393 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import datetime +import logging +import os +import stat + +log = logging.getLogger(__name__) + +import cdist.path + +CODE_HEADER = "#!/bin/sh -e\n" + +class Config: + """Cdist main class to hold arbitrary data""" + + def __init__(self, target_host, + initial_manifest=False, remote_user="root", + home=None, debug=False): + + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] + + self.path = cdist.path.Path(self.target_host, + initial_manifest=initial_manifest, + remote_user=self.remote_user, + remote_prefix=self.remote_prefix, + base_dir=home, + debug=debug) + + self.objects_prepared = [] + + def cleanup(self): + self.path.cleanup() + + def run_global_explores(self): + """Run global explorers""" + explorers = self.path.list_global_explorers() + if(len(explorers) == 0): + raise CdistError("No explorers found in", self.path.global_explorer_dir) + + self.path.transfer_global_explorers() + for explorer in explorers: + output = self.path.global_explorer_output_path(explorer) + output_fd = open(output, mode='w') + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append(self.path.remote_global_explorer_path(explorer)) + + cdist.exec.run_or_fail(cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def run_type_explorer(self, cdist_object): + """Run type specific explorers for objects""" + + type = self.path.get_type_from_object(cdist_object) + self.path.transfer_type_explorers(type) + + cmd = [] + cmd.append("__explorer=" + cdist.path.REMOTE_GLOBAL_EXPLORER_DIR) + cmd.append("__type_explorer=" + self.path.remote_type_explorer_dir(type)) + cmd.append("__object=" + self.path.remote_object_dir(cdist_object)) + cmd.append("__object_id=" + self.path.get_object_id_from_object(cdist_object)) + cmd.append("__object_fq=" + cdist_object) + + # Need to transfer at least the parameters for objects to be useful + self.path.transfer_object_parameter(cdist_object) + + explorers = self.path.list_type_explorers(type) + for explorer in explorers: + remote_cmd = cmd + [os.path.join(self.path.remote_type_explorer_dir(type), explorer)] + output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) + output_fd = open(output, mode='w') + log.debug("%s exploring %s using %s storing to %s", + cdist_object, explorer, remote_cmd, output) + + cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) + output_fd.close() + + def init_deploy(self): + """Ensure the base directories are cleaned up""" + log.debug("Creating clean directory structure") + + self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) + self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + + def run_initial_manifest(self): + """Run the initial manifest""" + env = { "__manifest" : self.path.manifest_dir } + self.run_manifest(self.path.initial_manifest, extra_env=env) + + def run_type_manifest(self, cdist_object): + """Run manifest for a specific object""" + type = self.path.get_type_from_object(cdist_object) + manifest = self.path.type_dir(type, "manifest") + + log.debug("%s: Running %s", cdist_object, manifest) + if os.path.exists(manifest): + env = { "__object" : self.path.object_dir(cdist_object), + "__object_id": self.path.get_object_id_from_object(cdist_object), + "__object_fq": cdist_object, + "__type": self.path.type_dir(type) + } + self.run_manifest(manifest, extra_env=env) + + def run_manifest(self, manifest, extra_env=None): + """Run a manifest""" + log.debug("Running manifest %s, env=%s", manifest, extra_env) + env = os.environ.copy() + env['PATH'] = self.path.bin_dir + ":" + env['PATH'] + + # Information required in every manifest + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + + # Legacy stuff to make cdist-type-emulator work + env['__cdist_core_dir'] = os.path.join(self.path.base_dir, "core") + env['__cdist_local_base_dir'] = self.path.temp_dir + + # Submit information to new type emulator + + # Required for recording source + env['__cdist_manifest'] = manifest + + # Required to find types + env['__cdist_type_base_dir'] = self.path.type_base_dir + + # Other environment stuff + if extra_env: + env.update(extra_env) + + cdist.exec.shell_run_or_debug_fail(manifest, [manifest], env=env) + + def object_run(self, cdist_object, mode): + """Run gencode or code for an object""" + log.debug("Running %s from %s", mode, cdist_object) + file=os.path.join(self.path.object_dir(cdist_object), "require") + requirements = cdist.path.file_to_list(file) + type = self.path.get_type_from_object(cdist_object) + + for requirement in requirements: + log.debug("Object %s requires %s", cdist_object, requirement) + self.object_run(requirement, mode=mode) + + # + # Setup env Variable: + # + env = os.environ.copy() + env['__target_host'] = self.target_host + env['__global'] = self.path.out_dir + env["__object"] = self.path.object_dir(cdist_object) + env["__object_id"] = self.path.get_object_id_from_object(cdist_object) + env["__object_fq"] = cdist_object + env["__type"] = self.path.type_dir(type) + + if mode == "gencode": + paths = [ + self.path.type_dir(type, "gencode-local"), + self.path.type_dir(type, "gencode-remote") + ] + for bin in paths: + if os.path.isfile(bin): + # omit "gen" from gencode and use it for output base + outfile=os.path.join(self.path.object_dir(cdist_object), + os.path.basename(bin)[3:]) + + outfile_fd = open(outfile, "w") + + # Need to flush to ensure our write is done before stdout write + outfile_fd.write(CODE_HEADER) + outfile_fd.flush() + + cdist.exec.shell_run_or_debug_fail(bin, [bin], env=env, stdout=outfile_fd) + outfile_fd.close() + + status = os.stat(outfile) + + # Remove output if empty, else make it executable + if status.st_size == len(CODE_HEADER): + os.unlink(outfile) + else: + # Add header and make executable - identically to 0o700 + os.chmod(outfile, stat.S_IXUSR | stat.S_IRUSR | stat.S_IWUSR) + + # Mark object as changed + open(os.path.join(self.path.object_dir(cdist_object), "changed"), "w").close() + + + if mode == "code": + local_dir = self.path.object_dir(cdist_object) + remote_dir = self.path.remote_object_dir(cdist_object) + + bin = os.path.join(local_dir, "code-local") + if os.path.isfile(bin): + cdist.exec.run_or_fail([bin]) + + + local_remote_code = os.path.join(local_dir, "code-remote") + remote_remote_code = os.path.join(remote_dir, "code-remote") + if os.path.isfile(local_remote_code): + self.path.transfer_file(local_remote_code, remote_remote_code) + # FIXME: remote_prefix + cdist.exec.run_or_fail([remote_remote_code], remote_prefix=self.remote_prefix) + + def stage_prepare(self): + """Do everything for a deploy, minus the actual code stage""" + self.init_deploy() + self.run_global_explores() + self.run_initial_manifest() + + old_objects = [] + objects = self.path.list_objects() + + # Continue process until no new objects are created anymore + while old_objects != objects: + log.debug("Prepare stage") + old_objects = list(objects) + for cdist_object in objects: + if cdist_object in self.objects_prepared: + log.debug("Skipping rerun of object %s", cdist_object) + continue + else: + self.run_type_explorer(cdist_object) + self.run_type_manifest(cdist_object) + self.objects_prepared.append(cdist_object) + + objects = self.path.list_objects() + + def stage_run(self): + """The final (and real) step of deployment""" + log.debug("Actual run objects") + # Now do the final steps over the existing objects + for cdist_object in self.path.list_objects(): + log.debug("Run object: %s", cdist_object) + self.object_run(cdist_object, mode="gencode") + self.object_run(cdist_object, mode="code") + + def deploy_to(self): + """Mimic the old deploy to: Deploy to one host""" + log.info("Deploying to " + self.target_host) + time_start = datetime.datetime.now() + + self.stage_prepare() + self.stage_run() + + time_end = datetime.datetime.now() + duration = time_end - time_start + log.info("Finished run of %s in %s seconds", + self.target_host, + duration.total_seconds()) + + def deploy_and_cleanup(self): + """Do what is most often done: deploy & cleanup""" + self.deploy_to() + self.cleanup() + +def config(args): + """Configure remote system""" + process = {} + + time_start = datetime.datetime.now() + + for host in args.host: + c = Config(host, initial_manifest=args.manifest, home=args.cdist_home, debug=args.debug) + if args.parallel: + log.debug("Creating child process for %s", host) + process[host] = multiprocessing.Process(target=c.deploy_and_cleanup) + process[host].start() + else: + c.deploy_and_cleanup() + + if args.parallel: + for p in process.keys(): + log.debug("Joining %s", p) + process[p].join() + + time_end = datetime.datetime.now() + log.info("Total processing time for %s host(s): %s", len(args.host), + (time_end - time_start).total_seconds()) + +def install(args): + """Install remote system""" + process = {} + +def commandline(): + """Parse command line""" + # Construct parser others can reuse + parser = {} + # Options _all_ parsers have in common + parser['most'] = argparse.ArgumentParser(add_help=False) + parser['most'].add_argument('-d', '--debug', + help='Set log level to debug', action='store_true') + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) + parser['main'].add_argument('-V', '--version', + help='Show version', action='version', + version='%(prog)s ' + cdist.VERSION) + parser['sub'] = parser['main'].add_subparsers(title="Commands") + + # Banner + parser['banner'] = parser['sub'].add_parser('banner', + add_help=False) + parser['banner'].set_defaults(func=cdist.banner.banner) + + # Config and install (common stuff) + parser['configinstall'] = argparse.ArgumentParser(add_help=False) + parser['configinstall'].add_argument('host', nargs='+', + help='one or more hosts to operate on') + parser['configinstall'].add_argument('-c', '--cdist-home', + help='Change cdist home (default: .. from bin directory)', + action='store') + parser['configinstall'].add_argument('-i', '--initial-manifest', + help='Path to a cdist manifest', + dest='manifest', required=False) + parser['configinstall'].add_argument('-p', '--parallel', + help='Operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['configinstall'].add_argument('-s', '--sequential', + help='Operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + + # Config + parser['config'] = parser['sub'].add_parser('config', + parents=[parser['most'], parser['configinstall']]) + parser['config'].set_defaults(func=config) + + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['most'], parser['configinstall']]) + parser['install'].set_defaults(func=install) + + for p in parser: + parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" + + args = parser['main'].parse_args(sys.argv[1:]) + + # Most subcommands have --debug, so handle it here + if 'debug' in args: + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) + + args.func(args) + + +if __name__ == "__main__": + try: + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): + cdist_lib = os.environ["__cdist_python_lib"] + sys.path.insert(0, cdist_lib) + import cdist.emulator + cdist.emulator.emulator(sys.argv) + else: + cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), + '../lib')) + sys.path.insert(0, cdist_lib) + + import cdist + import cdist.banner + import cdist.exec + import cdist.path + + commandline() + except KeyboardInterrupt: + sys.exit(0) + except cdist.Error as e: + log.error(e) + sys.exit(1) From 27b4b9cd039f30ad71f7533da79f195c82c1fbdc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:53:09 +0200 Subject: [PATCH 049/107] also move out install and cleanup library path code Signed-off-by: Nico Schottelius --- bin/cdist | 16 ++++++---------- lib/cdist/install.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 10 deletions(-) create mode 100755 lib/cdist/install.py diff --git a/bin/cdist b/bin/cdist index 4dd4f971..5e1b96bf 100755 --- a/bin/cdist +++ b/bin/cdist @@ -33,17 +33,13 @@ import sys import tempfile log = logging.getLogger(__name__) -real_me = os.path.dirname(os.path.realpath(__file__)) -cdist_lib = os.path.abspath(os.path.join(real_me, '../lib')) -sys.path.insert(0, cdist_lib) +# Ensure our /lib/ is included into PYTHON_PATH +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')) +) TYPE_PREFIX = "__" -CODE_HEADER = "#!/bin/sh -e\n" - -def install(args): - """Install remote system""" - process = {} def commandline(): """Parse command line""" @@ -91,7 +87,7 @@ def commandline(): # Install parser['install'] = parser['sub'].add_parser('install', parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) + parser['install'].set_defaults(func=cdist.install.install) for p in parser: parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" @@ -106,7 +102,6 @@ def commandline(): args.func(args) - if __name__ == "__main__": try: logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') @@ -119,6 +114,7 @@ if __name__ == "__main__": import cdist.banner import cdist.config import cdist.exec + import cdist.install import cdist.path commandline() diff --git a/lib/cdist/install.py b/lib/cdist/install.py new file mode 100755 index 00000000..98b388ec --- /dev/null +++ b/lib/cdist/install.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging + +log = logging.getLogger(__name__) + +def install(args): + """Install remote system""" + process = {} + From fc1e0d9f1fbdead1ab0c548ef17ce94f64f53f0c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 11:55:59 +0200 Subject: [PATCH 050/107] remove exec bits from copying Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 0 lib/cdist/emulator.py | 0 lib/cdist/install.py | 0 3 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/cdist/config.py mode change 100755 => 100644 lib/cdist/emulator.py mode change 100755 => 100644 lib/cdist/install.py diff --git a/lib/cdist/config.py b/lib/cdist/config.py old mode 100755 new mode 100644 diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py old mode 100755 new mode 100644 diff --git a/lib/cdist/install.py b/lib/cdist/install.py old mode 100755 new mode 100644 From 6458499d2bda5eacf2f40da3515dd103cc9031e7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 12:02:24 +0200 Subject: [PATCH 051/107] J) Signed-off-by: Nico Schottelius --- bin/cdist | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/cdist b/bin/cdist index 5e1b96bf..f8596e67 100755 --- a/bin/cdist +++ b/bin/cdist @@ -36,8 +36,7 @@ log = logging.getLogger(__name__) # Ensure our /lib/ is included into PYTHON_PATH sys.path.insert(0, os.path.abspath( - os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')) -) + os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib'))) TYPE_PREFIX = "__" From 0cd24f1544f71c63d9e8aaab7b9425b8e53c9772 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 15:08:59 +0200 Subject: [PATCH 052/107] rename quickstart to tutorial Signed-off-by: Nico Schottelius --- doc/man/man7/{cdist-quickstart.text => cdist-tutorial.text} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/man/man7/{cdist-quickstart.text => cdist-tutorial.text} (100%) diff --git a/doc/man/man7/cdist-quickstart.text b/doc/man/man7/cdist-tutorial.text similarity index 100% rename from doc/man/man7/cdist-quickstart.text rename to doc/man/man7/cdist-tutorial.text From 597e1e73453318f7886c8eb6c63f0ab8e27bc011 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 15:15:36 +0200 Subject: [PATCH 053/107] remove obsolete cdist-deploy-stdin-to Signed-off-by: Nico Schottelius --- bin/cdist-deploy-stdin-to | 36 ------------------------- doc/man/man1/cdist-deploy-stdin-to.text | 30 --------------------- 2 files changed, 66 deletions(-) delete mode 100755 bin/cdist-deploy-stdin-to delete mode 100644 doc/man/man1/cdist-deploy-stdin-to.text diff --git a/bin/cdist-deploy-stdin-to b/bin/cdist-deploy-stdin-to deleted file mode 100755 index 391dd431..00000000 --- a/bin/cdist-deploy-stdin-to +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Use stdin as the manifest to deploy on the given host. -# - -. cdist-config -[ $# -eq 1 ] || __cdist_usage "" -set -eu - -__cdist_target_host="$1" -shift - -cat >> "$__cdist_tmp_file" - -chmod +x "$__cdist_tmp_file" - -export __cdist_manifest_init="$__cdist_tmp_file" -cdist-deploy-to "$__cdist_target_host" diff --git a/doc/man/man1/cdist-deploy-stdin-to.text b/doc/man/man1/cdist-deploy-stdin-to.text deleted file mode 100644 index 14f19478..00000000 --- a/doc/man/man1/cdist-deploy-stdin-to.text +++ /dev/null @@ -1,30 +0,0 @@ -cdist-deploy-stdin-to(1) -======================== -Steven Armstrong - - -NAME ----- -cdist-deploy-stdin-to - Deploy the configuration given on stdin to host - - -SYNOPSIS --------- -echo "__file /tmp/whatever" | cdist-deploy-stdin-to HOSTNAME - - -DESCRIPTION ------------ -Use stdin as the manifest for cdist-deploy-to. - - -SEE ALSO --------- -- cdist(7) -- cdist-deploy-to(1) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). From 996fa75c2eff3bd0ffe4c6a10b30447a953209b1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 15:18:55 +0200 Subject: [PATCH 054/107] cleanup cdist imports Signed-off-by: Nico Schottelius --- bin/cdist | 6 ------ 1 file changed, 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index f8596e67..d21fda40 100755 --- a/bin/cdist +++ b/bin/cdist @@ -21,16 +21,10 @@ # import argparse -import datetime import logging -import multiprocessing import os import re -import subprocess -import shutil -import stat import sys -import tempfile log = logging.getLogger(__name__) From ef197eebe330bcdfb1af07806b288b88c8b6cfd7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 15:42:27 +0200 Subject: [PATCH 055/107] move test to top level Signed-off-by: Nico Schottelius --- test.py | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++ test/cdist.py | 12 ----------- 2 files changed, 59 insertions(+), 12 deletions(-) create mode 100755 test.py delete mode 100644 test/cdist.py diff --git a/test.py b/test.py new file mode 100755 index 00000000..ad56df11 --- /dev/null +++ b/test.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import os +import sys +import unittest + +sys.path.insert(0, os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))) + +import cdist +import cdist.config +import cdist.exec + +class Exec(unittest.TestCase): + def test_local_success(self): + try: + cdist.exec.run_or_fail(["/bin/true"]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail(self): + try: + cdist.exec.run_or_fail(["/bin/false"]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertTrue(failed) + + + +if __name__ == '__main__': + unittest.main() diff --git a/test/cdist.py b/test/cdist.py deleted file mode 100644 index 3ccc69bc..00000000 --- a/test/cdist.py +++ /dev/null @@ -1,12 +0,0 @@ -import cdist -import unittest - - -class CdistGeneric(unittest.TestCase): - - def test_initial_manifest(self): - self.assertEqual(numeral, result) - - -if __name__ == '__main__': - unittest.main() From 4906f604f0d1da1968c06e5e9e7cd3ca88a4c75d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 15:42:39 +0200 Subject: [PATCH 056/107] raise cdist errors Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 09e4e8a4..f95ba941 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -24,6 +24,8 @@ import subprocess log = logging.getLogger(__name__) +import cdist + def shell_run_or_debug_fail(script, *args, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called @@ -50,11 +52,11 @@ def shell_run_or_debug_fail(script, *args, **kargs): print(script_fd.read()) script_fd.close() except IOError as error: - raise CdistError(str(error)) + raise cdist.Error(str(error)) - raise CdistError("Command failed (shell): " + " ".join(*args)) + raise cdist.Error("Command failed (shell): " + " ".join(*args)) except OSError as error: - raise CdistError(" ".join(*args) + ": " + error.args[1]) + raise cdist.Error(" ".join(*args) + ": " + error.args[1]) def run_or_fail(*args, **kargs): @@ -66,6 +68,6 @@ def run_or_fail(*args, **kargs): try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: - raise CdistError("Command failed: " + " ".join(*args)) + raise cdist.Error("Command failed: " + " ".join(*args)) except OSError as error: - raise CdistError(" ".join(*args) + ": " + error.args[1]) + raise cdist.Error(" ".join(*args) + ": " + error.args[1]) From 7882b4a3ac11b0866819f92b50e9231d11166aed Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:03:53 +0200 Subject: [PATCH 057/107] only print env, if existent Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index f95ba941..b0f98b56 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -38,7 +38,10 @@ def shell_run_or_debug_fail(script, *args, **kargs): del kargs["remote_prefix"] log.debug("Shell exec cmd: %s", args) - log.debug("Shell exec env: %s", kargs['env']) + + if 'env' in kargs: + log.debug("Shell exec env: %s", kargs['env']) + try: subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: From 24b8a57df51160463c501f92523303dcdf050269 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:04:27 +0200 Subject: [PATCH 058/107] write first tests for the exec module Signed-off-by: Nico Schottelius --- test.py | 44 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 35 insertions(+), 9 deletions(-) diff --git a/test.py b/test.py index ad56df11..b5b80dc9 100755 --- a/test.py +++ b/test.py @@ -23,6 +23,8 @@ import os import sys +import shutil +import tempfile import unittest sys.path.insert(0, os.path.abspath( @@ -33,6 +35,38 @@ import cdist.config import cdist.exec class Exec(unittest.TestCase): + def setUp(self): + """Create shell code and co.""" + + self.temp_dir = tempfile.mkdtemp() + self.shell_false = os.path.join(self.temp_dir, "shell_false") + self.shell_true = os.path.join(self.temp_dir, "shell_true") + + true_fd = open(self.shell_false, "w") + true_fd.writelines(["!/bin/sh", "/bin/true"]) + true_fd.close() + + false_fd = open(self.shell_false, "w") + false_fd.writelines(["!/bin/sh", "/bin/false"]) + false_fd.close() + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_local_success_shell(self): + try: + cdist.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) + + def test_local_fail_shell(self): + self.assertRaises(cdist.Error, cdist.exec.shell_run_or_debug_fail, + self.shell_false, [self.shell_false]) + def test_local_success(self): try: cdist.exec.run_or_fail(["/bin/true"]) @@ -44,15 +78,7 @@ class Exec(unittest.TestCase): self.assertFalse(failed) def test_local_fail(self): - try: - cdist.exec.run_or_fail(["/bin/false"]) - except cdist.Error: - failed = True - else: - failed = False - - self.assertTrue(failed) - + self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) if __name__ == '__main__': From 6e73572a9514583c32f6643ed4d21d09ab8cde74 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:21:56 +0200 Subject: [PATCH 059/107] easier check for remote_prefix Signed-off-by: Nico Schottelius --- lib/cdist/exec.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index b0f98b56..7985bdb3 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -26,16 +26,13 @@ log = logging.getLogger(__name__) import cdist -def shell_run_or_debug_fail(script, *args, **kargs): +def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): # Manually execute /bin/sh, because sh -e does what we want # and sh -c -e does not exit if /bin/false called args[0][:0] = [ "/bin/sh", "-e" ] - remote = False - if "remote_prefix" in kargs: - remote = True - args[0][:0] = kargs["remote_prefix"] - del kargs["remote_prefix"] + if remote_prefix: + args[0][:0] = remote_prefix log.debug("Shell exec cmd: %s", args) @@ -46,9 +43,9 @@ def shell_run_or_debug_fail(script, *args, **kargs): subprocess.check_call(*args, **kargs) except subprocess.CalledProcessError: log.error("Code that raised the error:\n") - if remote: - # FIXME: included in Path! - remote_cat(script) + if remote_prefix: + run_or_fail(["cat", script], remote_prefix=remote_prefix) + else: try: script_fd = open(script) @@ -62,10 +59,9 @@ def shell_run_or_debug_fail(script, *args, **kargs): raise cdist.Error(" ".join(*args) + ": " + error.args[1]) -def run_or_fail(*args, **kargs): - if "remote_prefix" in kargs: - args[0][:0] = kargs["remote_prefix"] - del kargs["remote_prefix"] +def run_or_fail(*args, remote_prefix=False, **kargs): + if remote_prefix: + args[0][:0] = remote_prefix log.debug("Exec: " + " ".join(*args)) try: From c687dbdc70951e15d0ba8e7adbd486e8c147a493 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 16:22:09 +0200 Subject: [PATCH 060/107] remove remote_cat, as it's only used in exec module Signed-off-by: Nico Schottelius --- lib/cdist/path.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 0fa753a8..277b1401 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -125,10 +125,6 @@ class Path: """Create directory on remote side""" cdist.exec.run_or_fail(["mkdir", "-p", directory], remote_prefix=self.remote_prefix) - def remote_cat(filename): - """Use cat on the remote side for output""" - cdist.exec.run_or_fail(["cat", filename], remote_prefix=self.remote_prefix) - def remove_remote_dir(self, destination): cdist.exec.run_or_fail(["rm", "-rf", destination], remote_prefix=self.remote_prefix) From 5e862c9ede969ec026dbf19e2a3ea158182a8ede Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 17:33:26 +0200 Subject: [PATCH 061/107] make cdist-tutorial compile Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 55 ++++++++++++-------------------- 1 file changed, 21 insertions(+), 34 deletions(-) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index e146f1a8..dfda5325 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -1,40 +1,23 @@ -#!/bin/sh -# -# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# Give the user an introduction into cdist -# +cdist-tutorial(7) +================= +Nico Schottelius -. cdist-config -set -eu -banner="cdist-quickstart>" -continue="Press enter to continue or ctrl-c to abort." -create_continue="Press enter to create the described files/directories" +NAME +---- +cdist-tutorial - a guided introduction into cdist -__prompt() -{ - echo -n "$banner" "$@" - read answer -} -################################################################################ +INTRODUCTION +------------ +This tutorial is aimed at people learning cdist and shows +typical approaches as well as gives an easy start into +the world of configuration management. + + +NOT MIGRATED +------------ + # Intro of quickstart # cat << eof @@ -305,6 +288,10 @@ That's it, this is the end of the cdist-quickstart. I hope you've got some impression on how cdist works, here are again some pointers on where to continue to read: -cdist(7), cdist-deploy-to(1), cdist-type(7), cdist-stages(7) eof + +SEE ALSO +-------- +cdist(1), cdist-type(7), cdist-stages(7) + From d5df740f17c2207fdf0e92181c2f74b6671fbe1b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Sep 2011 20:14:04 +0200 Subject: [PATCH 062/107] always compile documentation with UTF-8 encoding Signed-off-by: Nico Schottelius --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index bef4393b..68abafde 100755 --- a/build.sh +++ b/build.sh @@ -27,7 +27,7 @@ #set -e # Manpage and HTML -A2XM="a2x -f manpage --no-xmllint" +A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8" A2XH="a2x -f xhtml --no-xmllint" # Developer webbase From df84480fcc5efb0763f34b420b9568396a0e375a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:02:04 +0200 Subject: [PATCH 063/107] BUGFIX: emulator compared parameter with value Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index ecdfff25..a97959e6 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -105,11 +105,11 @@ def emulator(argv): sys.exit(1) else: param_fd = open(file, "r") - param_old = param_fd.readlines() + value_old = param_fd.readlines() param_fd.close() - if(param_old != param): - print("Parameter " + param + " differs: " + " ".join(param_old) + " vs. " + param) + if(value_old != value): + print("Parameter " + param + " differs: " + " ".join(value_old) + " vs. " + value) print("Sources: " + " ".join(old_object_source) + " and " + object_source) sys.exit(1) else: From 1cfb6bf5a8f6707b30e9239f69d94c06df446290 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:11:59 +0200 Subject: [PATCH 064/107] make manpage compile for __partition_msdos_apply Signed-off-by: Nico Schottelius --- conf/type/__partition_msdos_apply/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/type/__partition_msdos_apply/man.text b/conf/type/__partition_msdos_apply/man.text index 4d4f127c..6cc53b77 100644 --- a/conf/type/__partition_msdos_apply/man.text +++ b/conf/type/__partition_msdos_apply/man.text @@ -5,7 +5,7 @@ Steven Armstrong NAME ---- -cdist-type__partition_msdos_apply +cdist-type__partition_msdos_apply - Apply dos partition settings DESCRIPTION From 87937ecec92232fc8e4aa402c35b9cd0fb988121 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:12:11 +0200 Subject: [PATCH 065/107] also always use utf-8 for html Signed-off-by: Nico Schottelius --- build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sh b/build.sh index 68abafde..d20e0211 100755 --- a/build.sh +++ b/build.sh @@ -28,7 +28,7 @@ # Manpage and HTML A2XM="a2x -f manpage --no-xmllint -a encoding=UTF-8" -A2XH="a2x -f xhtml --no-xmllint" +A2XH="a2x -f xhtml --no-xmllint -a encoding=UTF-8" # Developer webbase WEBDIR=$HOME/niconetz From 877d8fed309fac3df8691f4533f0ee5385b5a406 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:34:41 +0200 Subject: [PATCH 066/107] verify that corrupted manifests are detected as such Signed-off-by: Nico Schottelius --- test.py | 49 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/test.py b/test.py index b5b80dc9..d1d62d25 100755 --- a/test.py +++ b/test.py @@ -42,12 +42,12 @@ class Exec(unittest.TestCase): self.shell_false = os.path.join(self.temp_dir, "shell_false") self.shell_true = os.path.join(self.temp_dir, "shell_true") - true_fd = open(self.shell_false, "w") - true_fd.writelines(["!/bin/sh", "/bin/true"]) + true_fd = open(self.shell_true, "w") + true_fd.writelines(["#!/bin/sh", "/bin/true"]) true_fd.close() false_fd = open(self.shell_false, "w") - false_fd.writelines(["!/bin/sh", "/bin/false"]) + false_fd.writelines(["#!/bin/sh", "/bin/false"]) false_fd.close() def tearDown(self): @@ -55,7 +55,7 @@ class Exec(unittest.TestCase): def test_local_success_shell(self): try: - cdist.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) + cdist.exec.shell_run_or_debug_fail(self.shell_true, [self.shell_true]) except cdist.Error: failed = True else: @@ -80,6 +80,47 @@ class Exec(unittest.TestCase): def test_local_fail(self): self.assertRaises(cdist.Error, cdist.exec.run_or_fail, ["/bin/false"]) +class Config(unittest.TestCase): + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.init_manifest = os.path.join(self.temp_dir, "manifest") + self.config = cdist.config.Config("localhost", + initial_manifest=self.init_manifest) + + def test_initial_manifest_different_parameter(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir + "--mode 0700", + "__file " + self.temp_dir + "--mode 0600", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + + def test_initial_manifest_parameter_added(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir, + "__file " + self.temp_dir + "--mode 0600", + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + + def test_initial_manifest_parameter_removed(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir + "--mode 0600", + "__file " + self.temp_dir, + ]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + +# Todo: +# fail if parameter in manifest given are different +# fail if parameter in manifest given are absent once/given once +# succeed if same parameter is specified twice if __name__ == '__main__': unittest.main() From 603f1c3ae075c7d5e47f1bfd472e89cec884cda4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:38:08 +0200 Subject: [PATCH 067/107] also check that giving a paramter twice works Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 65 +++++++++++++++++++++++++++++++- test.py | 20 ++++++++-- 2 files changed, 79 insertions(+), 6 deletions(-) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index dfda5325..8abcbad4 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -15,8 +15,69 @@ typical approaches as well as gives an easy start into the world of configuration management. -NOT MIGRATED ------------- + +QUICK START +----------- +For those who just want to configure a system with the +cdist configuration management and do not need (or want) +to understand everything. + +Cdist uses **ssh** for communication and transportation +and usually logs into the **target host** as the +**root** user. So you need to configure the **ssh server** +of the target host to allow root logins: Edit +the file **/etc/ssh/sshd_config** and add one of the following +lines: + +-------------------------------------------------------------------------------- +# Allow login only via public key +PermitRootLogin without-password + +# Allow login via password and public key +PermitRootLogin yes +-------------------------------------------------------------------------------- + + +Before you can start using cdist, you need to ensure that +you can login +sshd config! + + + + + +You can copy and paste the following +code into your shell to get started and even configure your system. + +-------------------------------------------------------------------------------- +# Get cdist +git clone git://git.schottelius.org/cdist + +# Create manifest (maps configuration to host(s) +cd cdist +echo '__file /etc/cdist-configured' > conf/manifest/init +chmod 0700 conf/manifest/init + +echo 'Ensure that you can login as root to localhost without password' +echo '(i.e. via public key) and then press return' +read tmp + +# Configure localhost +./bin/cdist config localhost + +# Find out that cdist created /etc/cdist-configured +ls -l /etc/cdist-configured +-------------------------------------------------------------------------------- + +The file 'conf/manifest/init' is usually the entry point for cdist, +to find out what to configure on which host. All manifests are +essentially shell scripts. Every manifest can use the types known to +cdist, which are usually underline prefixed (\_\_). + + + +Everything you specify in manifests + # Intro of quickstart # diff --git a/test.py b/test.py index d1d62d25..2eb0b89f 100755 --- a/test.py +++ b/test.py @@ -117,10 +117,22 @@ class Config(unittest.TestCase): self.assertRaises(cdist.Error, self.config.run_initial_manifest()) -# Todo: -# fail if parameter in manifest given are different -# fail if parameter in manifest given are absent once/given once -# succeed if same parameter is specified twice + def test_initial_manifest_parameter_twice(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh", + "__file " + self.temp_dir + "--mode 0600", + "__file " + self.temp_dir + "--mode 0600", + ]) + manifest_fd.close() + + try: + self.config.run_initial_manifest() + except cdist.Error: + failed = True + else: + failed = False + + self.assertFalse(failed) if __name__ == '__main__': unittest.main() From 326ff6728365cd94b8e29d3dbbb49d0d704ebbe2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:48:29 +0200 Subject: [PATCH 068/107] break tests by fixing them Signed-off-by: Nico Schottelius --- test.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/test.py b/test.py index 2eb0b89f..a3652c41 100755 --- a/test.py +++ b/test.py @@ -89,9 +89,9 @@ class Config(unittest.TestCase): def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir + "--mode 0700", - "__file " + self.temp_dir + "--mode 0600", + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + "--mode 0700\n", + "__file " + self.temp_dir + "--mode 0600\n", ]) manifest_fd.close() @@ -99,9 +99,9 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_added(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir, - "__file " + self.temp_dir + "--mode 0600", + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + '\n', + "__file " + self.temp_dir + "--mode 0600\n", ]) manifest_fd.close() @@ -109,9 +109,9 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_removed(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir + "--mode 0600", - "__file " + self.temp_dir, + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + "\n", ]) manifest_fd.close() @@ -119,9 +119,9 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") - manifest_fd.writelines(["#!/bin/sh", - "__file " + self.temp_dir + "--mode 0600", - "__file " + self.temp_dir + "--mode 0600", + manifest_fd.writelines(["#!/bin/sh\n", + "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + "--mode 0600\n", ]) manifest_fd.close() From 2c0a7adf74578ce6151468afcf724ddddbcf46fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:49:12 +0200 Subject: [PATCH 069/107] replace CdistError with cdist.Error Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 3 ++- lib/cdist/exec.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index a97959e6..bd97f69b 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -24,6 +24,7 @@ import logging import os import sys +import cdist import cdist.path log = logging.getLogger(__name__) @@ -87,7 +88,7 @@ def emulator(argv): try: os.makedirs(param_out_dir, exist_ok=True) except OSError as error: - raise CdistError(param_out_dir + ": " + error.args[1]) + raise cdist.Error(param_out_dir + ": " + error.args[1]) # Record parameter params = vars(args) diff --git a/lib/cdist/exec.py b/lib/cdist/exec.py index 7985bdb3..9cedefcc 100644 --- a/lib/cdist/exec.py +++ b/lib/cdist/exec.py @@ -58,7 +58,6 @@ def shell_run_or_debug_fail(script, *args, remote_prefix=False, **kargs): except OSError as error: raise cdist.Error(" ".join(*args) + ": " + error.args[1]) - def run_or_fail(*args, remote_prefix=False, **kargs): if remote_prefix: args[0][:0] = remote_prefix From 1dcc3b771e42c7593f65890aa41fd723957693a7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 00:58:16 +0200 Subject: [PATCH 070/107] +some ssh hints, localhost hint Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index 8abcbad4..9b6e7492 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -14,6 +14,10 @@ This tutorial is aimed at people learning cdist and shows typical approaches as well as gives an easy start into the world of configuration management. +This tutorial assumes you are configuring **localhost**, because +it is always available. Just repace **localhost** with your target +host for real life usage. + QUICK START @@ -37,6 +41,19 @@ PermitRootLogin without-password PermitRootLogin yes -------------------------------------------------------------------------------- +As cdist uses ssh intensively, it is recommended to setup authentication +with public keys: + +-------------------------------------------------------------------------------- +# Generate pubkey pair as a normal user +ssh-keygen + +# Copy pubkey over to target host +ssh-copy-id root@target_host +-------------------------------------------------------------------------------- + +As soon as you are able to login to the target host + Before you can start using cdist, you need to ensure that you can login @@ -75,6 +92,9 @@ essentially shell scripts. Every manifest can use the types known to cdist, which are usually underline prefixed (\_\_). +SSH HINTS +--------- +Control master, ssh agent Everything you specify in manifests From d87deba30eccd70ce37b9b4844e2a25e873d9513 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:01:12 +0200 Subject: [PATCH 071/107] cleanup tutorial, fillup todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 280 ++++++++++++++++++++++++++++ doc/man/man7/cdist-tutorial.text | 301 +------------------------------ 2 files changed, 283 insertions(+), 298 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 4e07dd96..91b418a2 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,6 +8,286 @@ - and that ssh will wait for answer of prompt - nasty if used in parallel mode (scroll up!) + +SSH HINTS +--------- +Control master, ssh agent + +Everything you specify in manifests + + +# Intro of quickstart +# +cat << eof +$banner cdist version $__cdist_version + +Welcome to the interactive guide to cdist! +This is the interactive tutorial and beginners help for cdist and here's +our schedule: + + - Stages: How cdist operates + - Explorer: Explore facts of the target host + - Manifest: Map configurations to hosts + - Types: Bundled functionality + - Deploy a configuration to the local host! + +eof +__prompt "$continue" + +################################################################################ +# Stages +# +cat << eof + +To deploy configurations to a host, you call + + cdist-deploy-to + +which makes calls to other scripts, which realise the so called "stages". +Usually you'll not notice this, but in case you want to debug or hack cdist, +you can run each stage on its own. Besides that, you just need to remember +that the command cdist-deploy-to is the main cdist command. + +See also: + + Source of cdist-deploy-to(1), cdist-stages(7) + +eof +__prompt "$continue" + +################################################################################ +# Explorer +# +cat << eof + +The first thing cdist always does is running different explorers on the +target host. The explorers can be found in the directory + + ${__cdist_explorer_dir} + +An explorer is executed on the target host and its output is saved to a file. +You can use these files later to decide what or how to configure the host. + +For a demonstration, we'll call the OS explorer locally now, but remember: +This is only for demonstration, normally it is run on the target host. +The os explorer will which either displays the detected operating system or +nothing if it does not know your OS. + +See also: + + cdist-explorer(7) + +eof +explorer="${__cdist_explorer_dir}/os" + +__prompt "Press enter to execute $explorer" + +set -x +"$explorer" +set +x + +################################################################################ +# Manifest +# +cat << eof + +The initial manifest is the entry point for cdist to find out, what you would +like to have configured. It is located at + + ${__cdist_manifest_init} + +And can be as simple as + +-------------------------------------------------------------------------------- +__file /etc/cdist-configured --type file +-------------------------------------------------------------------------------- + +See also: + + cdist-manifest(7) + +eof +__prompt "$continue" + +cat << eof + +Let's take a deeper look at the initial manifest to understand what it means: + + __file /etc/cdist-configured --type file + | | | \\ + | | The parameter type \\ With the value file + | | + | | + | | This is the object id + | + __file is a so called "type" + + +This essentially looks like a standard command executed in the shell. +eof +__prompt "$continue" + +cat << eof + +And that's exactly true. Manifests are shell snippets that can use +types as commands with arguments. cdist prepends a special path +that contain links to the cdist-type-emulator, to \$PATH, so you +can use your types as a command. + +This is also the reason why types should always be prefixed with +"__", to prevent collisions with existing binaries. + +The object id is unique per type and used to prevent you from creating +the same object twice. + +Parameters are type specific and are always specified as --parameter . + +See also: + + cdist-type-build-emulation(1), cdist-type-emulator(1) + +eof +__prompt "$continue" + +################################################################################ +# Types +# +cat << eof + +Types are bundled functionality and are the main component of cdist. +If you want to have a feature x, you write the type __x. Types are stored in + + ${__cdist_type_dir} + +And cdist ships with some types already! + +See also: + + cdist-type(7) + +eof +__prompt "Press enter to see available types" + +set -x +ls ${__cdist_type_dir} +set +x + +cat << eof + +Types consist of the following parts: + + - ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional} + - ${__cdist_name_manifest} + - ${__cdist_name_explorer} + - ${__cdist_name_gencode} + +eof +__prompt "$continue" + + +cat << eof + +Every type must have a directory named ${__cdist_name_parameter}, which +contains required or optional parameters (in newline seperated files). + +If an object of a specific type was created in the initial manifest, +the manifest of the type is run and may create other objects. + +A type may have ${__cdist_name_explorer}, which are very similar to the +${__cdist_name_explorer} seen above, but with a different purpose: +They are specific to the type and are not relevant for other types. + +You may use them for instance to find out details on the target host, +so you can decide what to do on the target host eventually. + +After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of +a type have been run, ${__cdist_name_gencode} is executed, which creates +code to be executed on the target on stdout. + +eof +__prompt "$continue" + +################################################################################ +# Deployment +# + +cat << eof + +Now you've got some basic knowledge about cdist, let's configure your a host! + +Ensure that you have a ssh server running on the host and that you can login as root. + +eof + +__prompt "Enter hostname or press enter for localhost: " + +if [ "$answer" ]; then + host="$answer" +else + host="localhost" +fi + +manifestinit="conf/manifest/init" +cat << eof + +I'll now setup $manifestinit, containing the following code: + +-------------------------------------------------------------------------------- +# Every machine becomes a marker, so sysadmins know that automatic +# configurations are happening +__file /etc/cdist-configured + +case "\$__target_host" in + $host) + __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic + __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" + ;; +esac +-------------------------------------------------------------------------------- + +WARNING: This will overwrite ${manifestinit}. + +eof + +cat > "$__cdist_abs_mydir/../$manifestinit" << eof + +# Every machine becomes a marker, so sysadmins know that automatic +# configurations are happening +__file /etc/cdist-configured + +case "\$__target_host" in + $host) + __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic + __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" + ;; +esac + +eof + +chmod u+x "$__cdist_abs_mydir/../$manifestinit" + +cmd="cdist-deploy-to $host" + +__prompt "Press enter to run \"$cmd\"" + +# No quotes, we need field splitting +$cmd + +################################################################################ +# End +# + +cat << eof + + +-------------------------------------------------------------------------------- +That's it, this is the end of the cdist-quickstart. + +I hope you've got some impression on how cdist works, here are again some +pointers on where to continue to read: + + +eof -------------------------------------------------------------------------------- - Initial install support diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index 9b6e7492..15f2a81d 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -52,19 +52,9 @@ ssh-keygen ssh-copy-id root@target_host -------------------------------------------------------------------------------- -As soon as you are able to login to the target host - - -Before you can start using cdist, you need to ensure that -you can login -sshd config! - - - - - -You can copy and paste the following -code into your shell to get started and even configure your system. +As soon as you are able to login without passwort to the target host, +we can use cdist, to configure it. You can copy and paste the following +code into your shell to get started and configure localhost: -------------------------------------------------------------------------------- # Get cdist @@ -75,10 +65,6 @@ cd cdist echo '__file /etc/cdist-configured' > conf/manifest/init chmod 0700 conf/manifest/init -echo 'Ensure that you can login as root to localhost without password' -echo '(i.e. via public key) and then press return' -read tmp - # Configure localhost ./bin/cdist config localhost @@ -92,287 +78,6 @@ essentially shell scripts. Every manifest can use the types known to cdist, which are usually underline prefixed (\_\_). -SSH HINTS ---------- -Control master, ssh agent - -Everything you specify in manifests - - -# Intro of quickstart -# -cat << eof -$banner cdist version $__cdist_version - -Welcome to the interactive guide to cdist! -This is the interactive tutorial and beginners help for cdist and here's -our schedule: - - - Stages: How cdist operates - - Explorer: Explore facts of the target host - - Manifest: Map configurations to hosts - - Types: Bundled functionality - - Deploy a configuration to the local host! - -eof -__prompt "$continue" - -################################################################################ -# Stages -# -cat << eof - -To deploy configurations to a host, you call - - cdist-deploy-to - -which makes calls to other scripts, which realise the so called "stages". -Usually you'll not notice this, but in case you want to debug or hack cdist, -you can run each stage on its own. Besides that, you just need to remember -that the command cdist-deploy-to is the main cdist command. - -See also: - - Source of cdist-deploy-to(1), cdist-stages(7) - -eof -__prompt "$continue" - -################################################################################ -# Explorer -# -cat << eof - -The first thing cdist always does is running different explorers on the -target host. The explorers can be found in the directory - - ${__cdist_explorer_dir} - -An explorer is executed on the target host and its output is saved to a file. -You can use these files later to decide what or how to configure the host. - -For a demonstration, we'll call the OS explorer locally now, but remember: -This is only for demonstration, normally it is run on the target host. -The os explorer will which either displays the detected operating system or -nothing if it does not know your OS. - -See also: - - cdist-explorer(7) - -eof -explorer="${__cdist_explorer_dir}/os" - -__prompt "Press enter to execute $explorer" - -set -x -"$explorer" -set +x - -################################################################################ -# Manifest -# -cat << eof - -The initial manifest is the entry point for cdist to find out, what you would -like to have configured. It is located at - - ${__cdist_manifest_init} - -And can be as simple as - --------------------------------------------------------------------------------- -__file /etc/cdist-configured --type file --------------------------------------------------------------------------------- - -See also: - - cdist-manifest(7) - -eof -__prompt "$continue" - -cat << eof - -Let's take a deeper look at the initial manifest to understand what it means: - - __file /etc/cdist-configured --type file - | | | \\ - | | The parameter type \\ With the value file - | | - | | - | | This is the object id - | - __file is a so called "type" - - -This essentially looks like a standard command executed in the shell. -eof -__prompt "$continue" - -cat << eof - -And that's exactly true. Manifests are shell snippets that can use -types as commands with arguments. cdist prepends a special path -that contain links to the cdist-type-emulator, to \$PATH, so you -can use your types as a command. - -This is also the reason why types should always be prefixed with -"__", to prevent collisions with existing binaries. - -The object id is unique per type and used to prevent you from creating -the same object twice. - -Parameters are type specific and are always specified as --parameter . - -See also: - - cdist-type-build-emulation(1), cdist-type-emulator(1) - -eof -__prompt "$continue" - -################################################################################ -# Types -# -cat << eof - -Types are bundled functionality and are the main component of cdist. -If you want to have a feature x, you write the type __x. Types are stored in - - ${__cdist_type_dir} - -And cdist ships with some types already! - -See also: - - cdist-type(7) - -eof -__prompt "Press enter to see available types" - -set -x -ls ${__cdist_type_dir} -set +x - -cat << eof - -Types consist of the following parts: - - - ${__cdist_name_parameter} (${__cdist_name_parameter_required}/${__cdist_name_parameter_optional} - - ${__cdist_name_manifest} - - ${__cdist_name_explorer} - - ${__cdist_name_gencode} - -eof -__prompt "$continue" - - -cat << eof - -Every type must have a directory named ${__cdist_name_parameter}, which -contains required or optional parameters (in newline seperated files). - -If an object of a specific type was created in the initial manifest, -the manifest of the type is run and may create other objects. - -A type may have ${__cdist_name_explorer}, which are very similar to the -${__cdist_name_explorer} seen above, but with a different purpose: -They are specific to the type and are not relevant for other types. - -You may use them for instance to find out details on the target host, -so you can decide what to do on the target host eventually. - -After the ${__cdist_name_manifest} and the ${__cdist_name_explorer} of -a type have been run, ${__cdist_name_gencode} is executed, which creates -code to be executed on the target on stdout. - -eof -__prompt "$continue" - -################################################################################ -# Deployment -# - -cat << eof - -Now you've got some basic knowledge about cdist, let's configure your a host! - -Ensure that you have a ssh server running on the host and that you can login as root. - -eof - -__prompt "Enter hostname or press enter for localhost: " - -if [ "$answer" ]; then - host="$answer" -else - host="localhost" -fi - -manifestinit="conf/manifest/init" -cat << eof - -I'll now setup $manifestinit, containing the following code: - --------------------------------------------------------------------------------- -# Every machine becomes a marker, so sysadmins know that automatic -# configurations are happening -__file /etc/cdist-configured - -case "\$__target_host" in - $host) - __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic - __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" - ;; -esac --------------------------------------------------------------------------------- - -WARNING: This will overwrite ${manifestinit}. - -eof - -cat > "$__cdist_abs_mydir/../$manifestinit" << eof - -# Every machine becomes a marker, so sysadmins know that automatic -# configurations are happening -__file /etc/cdist-configured - -case "\$__target_host" in - $host) - __link /tmp/cdist-testfile --source /etc/cdist-configured --type symbolic - __addifnosuchline /tmp/cdist-welcome --line "Welcome to cdist" - ;; -esac - -eof - -chmod u+x "$__cdist_abs_mydir/../$manifestinit" - -cmd="cdist-deploy-to $host" - -__prompt "Press enter to run \"$cmd\"" - -# No quotes, we need field splitting -$cmd - -################################################################################ -# End -# - -cat << eof - - --------------------------------------------------------------------------------- -That's it, this is the end of the cdist-quickstart. - -I hope you've got some impression on how cdist works, here are again some -pointers on where to continue to read: - - -eof - SEE ALSO -------- cdist(1), cdist-type(7), cdist-stages(7) - From cdd4e0968343cc72d79f3228fd7e2f56c611c404 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:12:09 +0200 Subject: [PATCH 072/107] prefix/localhost in tutorial Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-tutorial.text | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text index 15f2a81d..80135da9 100755 --- a/doc/man/man7/cdist-tutorial.text +++ b/doc/man/man7/cdist-tutorial.text @@ -49,7 +49,7 @@ with public keys: ssh-keygen # Copy pubkey over to target host -ssh-copy-id root@target_host +ssh-copy-id root@localhost -------------------------------------------------------------------------------- As soon as you are able to login without passwort to the target host, @@ -75,7 +75,7 @@ ls -l /etc/cdist-configured The file 'conf/manifest/init' is usually the entry point for cdist, to find out what to configure on which host. All manifests are essentially shell scripts. Every manifest can use the types known to -cdist, which are usually underline prefixed (\_\_). +cdist, which are usually underline prefixed (__). SEE ALSO From 3e49b12a6d30dd4621aa5150948c3662cc18aaf0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:17:32 +0200 Subject: [PATCH 073/107] update cdist-hacker Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-hacker.text | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/doc/man/man7/cdist-hacker.text b/doc/man/man7/cdist-hacker.text index efd6ef7d..b9f79d01 100644 --- a/doc/man/man7/cdist-hacker.text +++ b/doc/man/man7/cdist-hacker.text @@ -26,26 +26,12 @@ in the latest version, drop a mail to the cdist mailing list, subject prefixed with "[BUG] ". -UNDERSTANDING CDIST INTERNALS ------------------------------ -IF you are interested in how cdist internally works, you can open -bin/cdist-config and bin/cdist-deploy-to in your favorite editor and -read the scripts bin/cdist-deploy-to calls. The magnificent HACKERS_README -may be of great help as well. - - CODING CONVENTIONS (EVERYWHERE) ------------------------------- If something should be better done or needs to fixed, add the word FIXME nearby, so grepping for FIXME gives all positions that need to be fixed. -CODING CONVENTIONS (CORE) -------------------------- -- All variables exported by cdist are prefixed with a double underscore (__) -- All cdist-internal variables are prefixed with __cdist_ and are generally not exported. - - HOW TO SUBMIT STUFF FOR INCLUSION INTO UPSTREAM CDIST ----------------------------------------------------- If you did some cool changes to cdist, which you value as a benefit for From ae4d6002f06364a12598d449262c3872746bc7b2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:21:16 +0200 Subject: [PATCH 074/107] permissions/rephrase Signed-off-by: Nico Schottelius --- doc/man/man7/cdist-explorer.text | 2 +- doc/man/man7/cdist-tutorial.text | 0 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100755 => 100644 doc/man/man7/cdist-tutorial.text diff --git a/doc/man/man7/cdist-explorer.text b/doc/man/man7/cdist-explorer.text index 63c7a5c9..e1909ab5 100644 --- a/doc/man/man7/cdist-explorer.text +++ b/doc/man/man7/cdist-explorer.text @@ -26,7 +26,7 @@ $__explorer/ (general and type explorer) or $__type_explorer/ (type explorer). In case of significant errors, the explorer may exit non-zero and return an -error message on stderr, which will cause the cdist run to abort. +error message on stderr, which will cause cdist to abort. You can also use stderr for debugging purposes while developing a new explorer. diff --git a/doc/man/man7/cdist-tutorial.text b/doc/man/man7/cdist-tutorial.text old mode 100755 new mode 100644 From 75023a4ac9061f0eca8792622e972d5d4b57e124 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:22:45 +0200 Subject: [PATCH 075/107] begin to cleanup cdist-stages Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 1 + doc/man/man7/cdist-stages.text | 35 +--------------------------------- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 91b418a2..14098099 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,6 +8,7 @@ - and that ssh will wait for answer of prompt - nasty if used in parallel mode (scroll up!) +- rewrite cdist-stages SSH HINTS --------- diff --git a/doc/man/man7/cdist-stages.text b/doc/man/man7/cdist-stages.text index 294dffc7..8ac30015 100644 --- a/doc/man/man7/cdist-stages.text +++ b/doc/man/man7/cdist-stages.text @@ -32,11 +32,6 @@ explorers. Every existing explorer is run on the target and the output of all explorers are copied back into the local cache. The results can be used by manifests and types. -Related documentation: - - cdist-explorer-run-global(1) - - cdist-remote-explorer-run(1) - - cdist-explorer(7) - STAGE 2: RUN THE INITIAL MANIFEST --------------------------------- @@ -46,11 +41,6 @@ the objects as defined in the manifest for the specific host. In this stage, no conflicts may occur, i.e. no object of the same type with the same id may be created. -Related documentation: - - cdist-manifest-run-init(1) - - cdist-manifest-run(1) - - cdist-manifest(7) - STAGE 3: OBJECT INFORMATION RETRIEVAL ------------------------------------- @@ -59,12 +49,6 @@ transfered to the target host and executed. The results are transfered back and can be used in the following stages to decide what changes need to be made on the target to implement the desired state. -Related documentation: - - cdist-object-explorer-run(1) - - cdist-remote-explorer-run(1) - - cdist-type(7) - - cdist-explorer(7) - STAGE 4: RUN THE OBJECT MANIFEST -------------------------------- @@ -79,11 +63,6 @@ The newly created objects are merged back into the existing tree. No conflicts may occur during the merge. A conflict would mean that two different objects try to create the same object, which indicates a broken configuration. -Related documentation: - - cdist-object-manifest-run(1) - - cdist-manifest-run(1) - - cdist-type(7) - STAGE 5: CODE GENERATION ------------------------ @@ -92,29 +71,17 @@ gencode scripts. The gencode scripts generate the code to be executed on the target on stdout. If the gencode executables fail, they must print diagnostic messages on stderr and exit non-zero. -Related documentation: - - cdist-object-gencode-run(1) - - cdist-object-gencode(1) - - cdist-type(7) - STAGE 6: CODE EXECUTION ----------------------- For every object the resulting code from the previous stage is transferred to the target host and executed there to apply the configuration changes. -Related documentation: - - cdist-object-code-run(1) - - cdist-code-run(1) - STAGE 7: CACHE -------------- The cache stores the information from the current run for later use. -Related documentation: - - cdist-cache(1) - SUMMARY ------- @@ -126,8 +93,8 @@ in correct order. SEE ALSO -------- +- cdist(1) - cdist(7) -- cdist-deploy-to(1) - cdist-reference(7) From 4879f744c168b536843764a1e40a99a6821ed279 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 01:25:04 +0200 Subject: [PATCH 076/107] manpage cleanups Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 +++- doc/man/man7/cdist-type.text | 6 +----- doc/man/man7/cdist.text | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index 14098099..b7748949 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -8,7 +8,9 @@ - and that ssh will wait for answer of prompt - nasty if used in parallel mode (scroll up!) -- rewrite cdist-stages +- rewrite cdist-stages, remove +- update man7! +- exec flag is not true for manifest anymore SSH HINTS --------- diff --git a/doc/man/man7/cdist-type.text b/doc/man/man7/cdist-type.text index 1af386fb..2439876c 100644 --- a/doc/man/man7/cdist-type.text +++ b/doc/man/man7/cdist-type.text @@ -35,10 +35,6 @@ __file /etc/cdist-configured --type file __package tree --state installed -------------------------------------------------------------------------------- -Internally cdist-type-emulator(1) will be called from cdist-manifest-run(1) to -save the given parameters into a cconfig database, so they can be accessed by -the manifest and gencode scripts of the type (see below). - A list of supported types can be found in the cdist-reference(7) manpage. SINGLETON TYPES @@ -111,7 +107,7 @@ __package_$type "$@" -------------------------------------------------------------------------------- As you can see, the type can reference different environment variables, -which are documented in cdist-environment-variables(7). +which are documented in cdist-reference(7). Always ensure the manifest is executable, otherwise cdist will not be able to execute it. diff --git a/doc/man/man7/cdist.text b/doc/man/man7/cdist.text index 9f7dbbab..2a5d1fe5 100644 --- a/doc/man/man7/cdist.text +++ b/doc/man/man7/cdist.text @@ -35,12 +35,11 @@ pull mechanism (client requests configuration). SEE ALSO -------- - Website: http://www.nico.schottelius.org/software/cdist/[] -- cdist-best-practise(7) -- cdist-deploy-to(1) - cdist-hacker(7) - cdist-manifest(7) -- cdist-quickstart(1) - cdist-type(7) +- cdist(1) +- cdist(7) COPYING From b9335bb7ce221abf09cc96951edd3ae2587d8ae9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 10:35:25 +0200 Subject: [PATCH 077/107] update os/os_version explorer to support owl (openwall linux) Signed-off-by: Nico Schottelius --- conf/explorer/os | 2 +- conf/explorer/os_version | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/conf/explorer/os b/conf/explorer/os index 3e1582ec..e59301e7 100755 --- a/conf/explorer/os +++ b/conf/explorer/os @@ -65,7 +65,7 @@ if [ -f /etc/SuSE-release ]; then exit 0 fi -if uname -r | grep -s '.owl' >/dev/null 2>&1; then +if [ -f /etc/owl-release ]; then echo owl exit 0 fi diff --git a/conf/explorer/os_version b/conf/explorer/os_version index 08fda60b..ef80e8fc 100755 --- a/conf/explorer/os_version +++ b/conf/explorer/os_version @@ -42,6 +42,9 @@ case "$($__explorer/os)" in *bsd|solaris) uname -r ;; + owl) + cat /etc/owl-release + ;; redhat|centos) cat /etc/redhat-release ;; From 6f1a13b531cc0664721b5763f8f2be17e8beb807 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 13:38:25 +0200 Subject: [PATCH 078/107] move emulator link to emulator module and make source variable (exec_path) Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 115 +++++++----------------------------------- lib/cdist/emulator.py | 9 ++++ lib/cdist/path.py | 10 ---- 3 files changed, 26 insertions(+), 108 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index a5a2252f..cc0f7817 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -24,9 +24,11 @@ import datetime import logging import os import stat +import sys log = logging.getLogger(__name__) +import cdist.emulator import cdist.path CODE_HEADER = "#!/bin/sh -e\n" @@ -35,12 +37,18 @@ class Config: """Cdist main class to hold arbitrary data""" def __init__(self, target_host, - initial_manifest=False, remote_user="root", - home=None, debug=False): + initial_manifest=False, + remote_user="root", + home=None, + exec_path=sys.argv[0], + debug=False): - self.target_host = target_host - self.debug = debug - self.remote_user = remote_user + self.target_host = target_host + self.debug = debug + self.remote_user = remote_user + self.exec_path = exec_path + + # FIXME: broken - construct elsewhere! self.remote_prefix = ["ssh", self.remote_user + "@" + self.target_host] self.path = cdist.path.Path(self.target_host, @@ -94,7 +102,7 @@ class Config: output = os.path.join(self.path.type_explorer_output_dir(cdist_object), explorer) output_fd = open(output, mode='w') log.debug("%s exploring %s using %s storing to %s", - cdist_object, explorer, remote_cmd, output) + cdist_object, explorer, remote_cmd, output) cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() @@ -106,6 +114,9 @@ class Config: self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + def run_initial_manifest(self): """Run the initial manifest""" env = { "__manifest" : self.path.manifest_dir } @@ -299,95 +310,3 @@ def config(args): time_end = datetime.datetime.now() log.info("Total processing time for %s host(s): %s", len(args.host), (time_end - time_start).total_seconds()) - -def install(args): - """Install remote system""" - process = {} - -def commandline(): - """Parse command line""" - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') - - # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) - parser['main'].add_argument('-V', '--version', - help='Show version', action='version', - version='%(prog)s ' + cdist.VERSION) - parser['sub'] = parser['main'].add_subparsers(title="Commands") - - # Banner - parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) - parser['banner'].set_defaults(func=cdist.banner.banner) - - # Config and install (common stuff) - parser['configinstall'] = argparse.ArgumentParser(add_help=False) - parser['configinstall'].add_argument('host', nargs='+', - help='one or more hosts to operate on') - parser['configinstall'].add_argument('-c', '--cdist-home', - help='Change cdist home (default: .. from bin directory)', - action='store') - parser['configinstall'].add_argument('-i', '--initial-manifest', - help='Path to a cdist manifest', - dest='manifest', required=False) - parser['configinstall'].add_argument('-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['configinstall'].add_argument('-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - - # Config - parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) - parser['config'].set_defaults(func=config) - - # Install - parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) - parser['install'].set_defaults(func=install) - - for p in parser: - parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" - - args = parser['main'].parse_args(sys.argv[1:]) - - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) - - args.func(args) - - -if __name__ == "__main__": - try: - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') - - if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): - cdist_lib = os.environ["__cdist_python_lib"] - sys.path.insert(0, cdist_lib) - import cdist.emulator - cdist.emulator.emulator(sys.argv) - else: - cdist_lib = os.path.abspath(os.path.join(os.path.dirname(__file__), - '../lib')) - sys.path.insert(0, cdist_lib) - - import cdist - import cdist.banner - import cdist.exec - import cdist.path - - commandline() - except KeyboardInterrupt: - sys.exit(0) - except cdist.Error as e: - log.error(e) - sys.exit(1) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index bd97f69b..b96b62dd 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -133,3 +133,12 @@ def emulator(argv): # sys.exit(1) print("Finished " + type + "/" + object_id + repr(params)) + + +def link(exec_path, bin_dir, type_list): + """Link type names to cdist-type-emulator""" + source = os.path.abspath(exec_path) + for type in type_list: + destination = os.path.join(bin_dir, type) + log.debug("Linking %s to %s", source, destination) + os.symlink(source, destination) diff --git a/lib/cdist/path.py b/lib/cdist/path.py index 277b1401..e416c42d 100644 --- a/lib/cdist/path.py +++ b/lib/cdist/path.py @@ -94,7 +94,6 @@ class Path: # Setup binary directory + contents self.bin_dir = os.path.join(self.out_dir, "bin") os.mkdir(self.bin_dir) - self.link_type_to_emulator() # List of type explorers transferred self.type_explorers_transferred = {} @@ -275,12 +274,3 @@ class Path: # Ensure that the path exists self.remote_mkdir(remote_base) self.transfer_dir(src, dst) - - - def link_type_to_emulator(self): - """Link type names to cdist-type-emulator""" - source = os.path.abspath(sys.argv[0]) - for type in self.list_types(): - destination = os.path.join(self.bin_dir, type) - log.debug("Linking %s to %s", source, destination) - os.symlink(source, destination) From 7ec9fbbf128e72b16a588007cd2d11d92198a673 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 14:01:04 +0200 Subject: [PATCH 079/107] fix argument passing in tests Signed-off-by: Nico Schottelius --- test.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/test.py b/test.py index a3652c41..908fad5e 100755 --- a/test.py +++ b/test.py @@ -30,6 +30,9 @@ import unittest sys.path.insert(0, os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), 'lib'))) +cdist_exec_path = os.path.abspath( + os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) + import cdist import cdist.config import cdist.exec @@ -85,13 +88,14 @@ class Config(unittest.TestCase): self.temp_dir = tempfile.mkdtemp() self.init_manifest = os.path.join(self.temp_dir, "manifest") self.config = cdist.config.Config("localhost", - initial_manifest=self.init_manifest) + initial_manifest=self.init_manifest, + exec_path=cdist_exec_path) def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + "--mode 0700\n", - "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + " --mode 0700\n", + "__file " + self.temp_dir + " --mode 0600\n", ]) manifest_fd.close() @@ -101,7 +105,7 @@ class Config(unittest.TestCase): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", "__file " + self.temp_dir + '\n', - "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", ]) manifest_fd.close() @@ -110,7 +114,7 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_removed(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", "__file " + self.temp_dir + "\n", ]) manifest_fd.close() @@ -120,8 +124,8 @@ class Config(unittest.TestCase): def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", - "__file " + self.temp_dir + "--mode 0600\n", - "__file " + self.temp_dir + "--mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", + "__file " + self.temp_dir + " --mode 0600\n", ]) manifest_fd.close() From 9d582ae24f1fa4c3697bb0ddfd83a146fcb75432 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 14:01:09 +0200 Subject: [PATCH 080/107] rename emulator to run Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- lib/cdist/emulator.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/cdist b/bin/cdist index d21fda40..245b2fc0 100755 --- a/bin/cdist +++ b/bin/cdist @@ -101,7 +101,7 @@ if __name__ == "__main__": if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): import cdist.emulator - cdist.emulator.emulator(sys.argv) + cdist.emulator.run(sys.argv) else: import cdist import cdist.banner diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index b96b62dd..33594e45 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -29,7 +29,7 @@ import cdist.path log = logging.getLogger(__name__) -def emulator(argv): +def run(argv): """Emulate type commands (i.e. __file and co)""" type = os.path.basename(argv[0]) type_dir = os.path.join(os.environ['__cdist_type_base_dir'], type) @@ -101,8 +101,8 @@ def emulator(argv): # Already exists, verify all parameter are the same if object_exists: if not os.path.isfile(file): - print("New parameter + " + param + "specified, aborting") - print("Source = " + old_object_source + "new =" + object_source) + print("New parameter + \"" + param + "\" specified, aborting") + print("Source = " + " ".join(old_object_source) + " new =" + object_source) sys.exit(1) else: param_fd = open(file, "r") From 0913bb21dd26ec871a683d77a9e623f2be3716f5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 14:08:11 +0200 Subject: [PATCH 081/107] use raise instead of sys.exit in emulator Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 33594e45..68a67176 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -22,7 +22,6 @@ import argparse import logging import os -import sys import cdist import cdist.path @@ -101,18 +100,22 @@ def run(argv): # Already exists, verify all parameter are the same if object_exists: if not os.path.isfile(file): - print("New parameter + \"" + param + "\" specified, aborting") - print("Source = " + " ".join(old_object_source) + " new =" + object_source) - sys.exit(1) + raise cdist.Error("New parameter \"" + + param + "\" specified, aborting\n" + + "Source = " + + " ".join(old_object_source) + + " new =" + object_source) else: param_fd = open(file, "r") value_old = param_fd.readlines() param_fd.close() if(value_old != value): - print("Parameter " + param + " differs: " + " ".join(value_old) + " vs. " + value) - print("Sources: " + " ".join(old_object_source) + " and " + object_source) - sys.exit(1) + raise cdist.Error("Parameter + \"" + param + + "\" differs: " + " ".join(value_old) + " vs. " + + value + + "\nSource = " + " ".join(old_object_source) + + " new =" + object_source) else: param_fd = open(file, "w") param_fd.writelines(value) @@ -131,7 +134,6 @@ def run(argv): source_fd.writelines(object_source) source_fd.close() - # sys.exit(1) print("Finished " + type + "/" + object_id + repr(params)) From cc87573d6b877fb7e49db8cea342204bd02efc18 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 15:35:04 +0200 Subject: [PATCH 082/107] remove lecagy cdist-type-emulator support Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index cc0f7817..aef0b28a 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -146,12 +146,6 @@ class Config: env['__target_host'] = self.target_host env['__global'] = self.path.out_dir - # Legacy stuff to make cdist-type-emulator work - env['__cdist_core_dir'] = os.path.join(self.path.base_dir, "core") - env['__cdist_local_base_dir'] = self.path.temp_dir - - # Submit information to new type emulator - # Required for recording source env['__cdist_manifest'] = manifest From dc416f5f4910ba9c8d80efdcc68e0717123d906e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 15:43:54 +0200 Subject: [PATCH 083/107] pass function, not return :-) Signed-off-by: Nico Schottelius --- test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test.py b/test.py index 908fad5e..44068d52 100755 --- a/test.py +++ b/test.py @@ -99,7 +99,7 @@ class Config(unittest.TestCase): ]) manifest_fd.close() - self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + self.assertRaises(cdist.Error, self.config.run_initial_manifest) def test_initial_manifest_parameter_added(self): manifest_fd = open(self.init_manifest, "w") @@ -109,7 +109,7 @@ class Config(unittest.TestCase): ]) manifest_fd.close() - self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + self.assertRaises(cdist.Error, self.config.run_initial_manifest) def test_initial_manifest_parameter_removed(self): manifest_fd = open(self.init_manifest, "w") @@ -119,7 +119,7 @@ class Config(unittest.TestCase): ]) manifest_fd.close() - self.assertRaises(cdist.Error, self.config.run_initial_manifest()) + self.assertRaises(cdist.Error, self.config.run_initial_manifest) def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") From 3ebece5784bf64f1e961a5d8fdae1a4da8d52507 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 15:47:56 +0200 Subject: [PATCH 084/107] +\n in writelines Signed-off-by: Nico Schottelius --- test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index 44068d52..8a797d98 100755 --- a/test.py +++ b/test.py @@ -46,11 +46,11 @@ class Exec(unittest.TestCase): self.shell_true = os.path.join(self.temp_dir, "shell_true") true_fd = open(self.shell_true, "w") - true_fd.writelines(["#!/bin/sh", "/bin/true"]) + true_fd.writelines(["#!/bin/sh\n", "/bin/true"]) true_fd.close() false_fd = open(self.shell_false, "w") - false_fd.writelines(["#!/bin/sh", "/bin/false"]) + false_fd.writelines(["#!/bin/sh\n", "/bin/false"]) false_fd.close() def tearDown(self): From 1d981200c8cfaff9da4a301a261525649bab2da7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 15:56:43 +0200 Subject: [PATCH 085/107] test breaking run, if a non existent command is called Signed-off-by: Nico Schottelius --- test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/test.py b/test.py index 8a797d98..1e9aee48 100755 --- a/test.py +++ b/test.py @@ -121,6 +121,14 @@ class Config(unittest.TestCase): self.assertRaises(cdist.Error, self.config.run_initial_manifest) + def test_initial_manifest_non_existent_command(self): + manifest_fd = open(self.init_manifest, "w") + manifest_fd.writelines(["#!/bin/sh\n", + "thereisdefinitelynosuchcommend"]) + manifest_fd.close() + + self.assertRaises(cdist.Error, self.config.run_initial_manifest) + def test_initial_manifest_parameter_twice(self): manifest_fd = open(self.init_manifest, "w") manifest_fd.writelines(["#!/bin/sh\n", From ccbd0f1d8450af180f5883f3b4dad0d429dc35d6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:05:26 +0200 Subject: [PATCH 086/107] introduce config.link_emulator() to be called from test Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 11 +++++++---- test.py | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index aef0b28a..05bdb2a1 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -107,15 +107,18 @@ class Config: cdist.exec.run_or_fail(remote_cmd, stdout=output_fd, remote_prefix=self.remote_prefix) output_fd.close() + def link_emulator(self): + """Link emulator to types""" + cdist.emulator.link(self.exec_path, + self.path.bin_dir, self.path.list_types()) + def init_deploy(self): """Ensure the base directories are cleaned up""" log.debug("Creating clean directory structure") self.path.remove_remote_dir(cdist.path.REMOTE_BASE_DIR) self.path.remote_mkdir(cdist.path.REMOTE_BASE_DIR) - - cdist.emulator.link(self.exec_path, - self.path.bin_dir, self.path.list_types()) + self.link_emulator() def run_initial_manifest(self): """Run the initial manifest""" @@ -149,7 +152,7 @@ class Config: # Required for recording source env['__cdist_manifest'] = manifest - # Required to find types + # Required to find types env['__cdist_type_base_dir'] = self.path.type_base_dir # Other environment stuff diff --git a/test.py b/test.py index 1e9aee48..ce433152 100755 --- a/test.py +++ b/test.py @@ -90,6 +90,7 @@ class Config(unittest.TestCase): self.config = cdist.config.Config("localhost", initial_manifest=self.init_manifest, exec_path=cdist_exec_path) + self.config.link_emulator() def test_initial_manifest_different_parameter(self): manifest_fd = open(self.init_manifest, "w") From 272b8722c1cdda9fa7f1ef349abe9cdc9366945d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:10:53 +0200 Subject: [PATCH 087/107] fix output, compare string with string not string with list Signed-off-by: Nico Schottelius --- lib/cdist/emulator.py | 6 +++--- test.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 68a67176..6a99ba79 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -110,12 +110,12 @@ def run(argv): value_old = param_fd.readlines() param_fd.close() - if(value_old != value): - raise cdist.Error("Parameter + \"" + param + + if(value_old[0] != value): + raise cdist.Error("Parameter\"" + param + "\" differs: " + " ".join(value_old) + " vs. " + value + "\nSource = " + " ".join(old_object_source) - + " new =" + object_source) + + " new = " + object_source) else: param_fd = open(file, "w") param_fd.writelines(value) diff --git a/test.py b/test.py index ce433152..b6a575e9 100755 --- a/test.py +++ b/test.py @@ -139,7 +139,9 @@ class Config(unittest.TestCase): manifest_fd.close() try: + print("a") self.config.run_initial_manifest() + print("b") except cdist.Error: failed = True else: From a1655856999a06fae352e61bf54d6b29561d58dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:20:46 +0200 Subject: [PATCH 088/107] pass __debug to manifest Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 5 ++++- lib/cdist/emulator.py | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 05bdb2a1..6df5d6d8 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -149,10 +149,13 @@ class Config: env['__target_host'] = self.target_host env['__global'] = self.path.out_dir + # Submit debug flag to manifest, can be used by emulator and types + env['__debug'] = "yes" + # Required for recording source env['__cdist_manifest'] = manifest - # Required to find types + # Required to find types env['__cdist_type_base_dir'] = self.path.type_base_dir # Other environment stuff diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 6a99ba79..9fc7c576 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -36,6 +36,9 @@ def run(argv): global_dir = os.environ['__global'] object_source = os.environ['__cdist_manifest'] + if '__debug' in os.environ: + logging.root.setLevel(logging.DEBUG) + parser = argparse.ArgumentParser(add_help=False) # Setup optional parameters @@ -134,7 +137,7 @@ def run(argv): source_fd.writelines(object_source) source_fd.close() - print("Finished " + type + "/" + object_id + repr(params)) + log.debug("Finished " + type + "/" + object_id + repr(params)) def link(exec_path, bin_dir, type_list): From b63fdf16edb71129908fedda0fd82e06d33d7b7a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:21:36 +0200 Subject: [PATCH 089/107] document __debug in reference Signed-off-by: Nico Schottelius --- doc/man/cdist-reference.text.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/doc/man/cdist-reference.text.sh b/doc/man/cdist-reference.text.sh index c205bdcc..7196c3b3 100755 --- a/doc/man/cdist-reference.text.sh +++ b/doc/man/cdist-reference.text.sh @@ -166,6 +166,11 @@ changed:: ENVIRONMENT VARIABLES --------------------- +__debug:: + If this variable is setup, cdist runs in debug mode. + You can use this information, to only output stuff in debug + mode as well. + Available for: initial manifest, type manifest __explorer:: Directory that contains all global explorers. Available for: explorer From ae7887f775b8db8b1f89f1798de22dc15f9aa98a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:25:58 +0200 Subject: [PATCH 090/107] ++changes Signed-off-by: Nico Schottelius --- doc/changelog | 2 ++ lib/cdist/emulator.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index a08efb34..53ea04a4 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,5 +1,7 @@ 2.0.2: * Add support for detection of OpenWall Linux (Matthias Teege) + * Add support for __debug variable in manifests + * Bugfix core: Various issues with type emulator 2.0.1: 2011-09-23 * Bugfix core: Always print source of error in case of exec errors diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index 9fc7c576..d69694df 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -127,7 +127,7 @@ def run(argv): # Record requirements if "__require" in os.environ: requirements = os.environ['__require'] - print(object_id + ":Writing requirements: " + requirements) + log.debug(object_id + ":Writing requirements: " + requirements) require_fd = open(os.path.join(object_dir, "require"), "a") require_fd.writelines(requirements.split(" ")) require_fd.close() From b52939ccfedf2b4e536f103cd883f61b19cf67a9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:40:50 +0200 Subject: [PATCH 091/107] only debug if __debug is setup Signed-off-by: Nico Schottelius --- lib/cdist/config.py | 3 ++- lib/cdist/emulator.py | 12 +++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/lib/cdist/config.py b/lib/cdist/config.py index 6df5d6d8..af47d091 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -150,7 +150,8 @@ class Config: env['__global'] = self.path.out_dir # Submit debug flag to manifest, can be used by emulator and types - env['__debug'] = "yes" + if self.debug: + env['__debug'] = "yes" # Required for recording source env['__cdist_manifest'] = manifest diff --git a/lib/cdist/emulator.py b/lib/cdist/emulator.py index d69694df..38a58f8c 100644 --- a/lib/cdist/emulator.py +++ b/lib/cdist/emulator.py @@ -38,21 +38,19 @@ def run(argv): if '__debug' in os.environ: logging.root.setLevel(logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) parser = argparse.ArgumentParser(add_help=False) - # Setup optional parameters for parameter in cdist.path.file_to_list(os.path.join(param_dir, "optional")): argument = "--" + parameter parser.add_argument(argument, action='store', required=False) - - # Setup required parameters for parameter in cdist.path.file_to_list(os.path.join(param_dir, "required")): argument = "--" + parameter parser.add_argument(argument, action='store', required=True) - # Setup positional parameter, if not singleton - + # If not singleton support one positional parameter if not os.path.isfile(os.path.join(type_dir, "singleton")): parser.add_argument("object_id", nargs=1) @@ -70,6 +68,10 @@ def run(argv): if object_id[0] == '/': object_id = object_id[1:] + # Prefix output by object_self + logformat = '%(levelname)s: ' + type + '/' + object_id + ': %(message)s' + logging.basicConfig(format=logformat) + # FIXME: verify object id log.debug(args) From 67f45944dd7d99b4a7710f34dc419ce760d65047 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:43:53 +0200 Subject: [PATCH 092/107] less todo Signed-off-by: Nico Schottelius --- doc/dev/todo/niconext | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/doc/dev/todo/niconext b/doc/dev/todo/niconext index b7748949..11c734f9 100644 --- a/doc/dev/todo/niconext +++ b/doc/dev/todo/niconext @@ -311,15 +311,13 @@ eof via __global/ - Support parallel execution - - and maximum number of parallel runs (-p X) - error handling / report failed hosts -- Allow manifest to be read from stdin - Create new video for cdist 2.0.0 http://www.youtube.com/watch?v=PRMjzy48eTI - Setup __debug, if -d is given, so other tools can reuse it - (-> non core feature! + - implement everywhere to external! - remote_prefix: scp vs. ssh issue From 4a0e0ceb8a9ec695ae5e0c7c0760dc5684d82d0f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:44:38 +0200 Subject: [PATCH 093/107] release 2.0.2 today Signed-off-by: Nico Schottelius --- doc/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changelog b/doc/changelog index 53ea04a4..39455351 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,4 +1,4 @@ -2.0.2: +2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) * Add support for __debug variable in manifests * Bugfix core: Various issues with type emulator From 6519d7e92c72ff66ee2fcaeeaa62e2fb45424269 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:53:02 +0200 Subject: [PATCH 094/107] add hint for debian/argparse Signed-off-by: Nico Schottelius --- README | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/README b/README index daa12f70..7c089cf3 100644 --- a/README +++ b/README @@ -86,13 +86,23 @@ cdist was tested or is know to run on at least * SSH-Server -## Getting cdist +## Installation + +### Preperation + +Ensure you have Python 3.x and the **argparse** module installed on +the machine you use to **deploy to the targets**. + +#### Debian + + aptitude install python3 python3-setuptools + easy_install3 argparse + + +### Get cdist You can clone cdist from git, which gives you the advantage of having a version control in place for development of your own stuff as well. - -### Installation - To install cdist, execute the following commands: git clone git://git.schottelius.org/cdist From dfc3d4c7c87bc6ce0e601940a6e4417f037684da Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 16:55:39 +0200 Subject: [PATCH 095/107] toc = level 3 (we are getting bigger) Signed-off-by: Nico Schottelius --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 7c089cf3..555d28d6 100644 --- a/README +++ b/README @@ -15,7 +15,7 @@ "P' "" "" -[[!toc levels=2]] +[[!toc levels=3]] ## Introduction From 6c9bf9e5d7d527f4e098272e68e5ad67d6de0322 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:12:08 +0200 Subject: [PATCH 096/107] use cdist, not __main__ as logger name Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index 245b2fc0..c79db293 100755 --- a/bin/cdist +++ b/bin/cdist @@ -26,7 +26,7 @@ import os import re import sys -log = logging.getLogger(__name__) +log = logging.getLogger("cdist") # Ensure our /lib/ is included into PYTHON_PATH sys.path.insert(0, os.path.abspath( From d8da7635def25dc9873886681345d67418e80de5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:15:54 +0200 Subject: [PATCH 097/107] use warning loglevel by default Signed-off-by: Nico Schottelius --- bin/cdist | 2 +- lib/cdist/config.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cdist b/bin/cdist index c79db293..ea7cf4f9 100755 --- a/bin/cdist +++ b/bin/cdist @@ -97,7 +97,7 @@ def commandline(): if __name__ == "__main__": try: - logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + logging.basicConfig(format='%(levelname)s: %(message)s') if re.match(TYPE_PREFIX, os.path.basename(sys.argv[0])): import cdist.emulator diff --git a/lib/cdist/config.py b/lib/cdist/config.py index af47d091..f50195aa 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -65,6 +65,7 @@ class Config: def run_global_explores(self): """Run global explorers""" + log.info("Running global explorers") explorers = self.path.list_global_explorers() if(len(explorers) == 0): raise CdistError("No explorers found in", self.path.global_explorer_dir) From 0e8dcb2f3db257e5e76f89b4b8394e3ceaa09a4b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:20:31 +0200 Subject: [PATCH 098/107] add verbose support Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/bin/cdist b/bin/cdist index ea7cf4f9..7d78ff51 100755 --- a/bin/cdist +++ b/bin/cdist @@ -41,7 +41,11 @@ def commandline(): # Options _all_ parsers have in common parser['most'] = argparse.ArgumentParser(add_help=False) parser['most'].add_argument('-d', '--debug', - help='Set log level to debug', action='store_true') + help='Set log level to debug', action='store_true', + default=False) + parser['most'].add_argument('-v', '--verbose', + help='Set log level to info, is more verbose', + action='store_true', default=False) # Main subcommand parser parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) @@ -87,12 +91,13 @@ def commandline(): args = parser['main'].parse_args(sys.argv[1:]) - # Most subcommands have --debug, so handle it here - if 'debug' in args: - if args.debug: - logging.root.setLevel(logging.DEBUG) - log.debug(args) + # Loglevels are handled globally in here and debug wins over verbose + if args.verbose: + logging.root.setLevel(logging.INFO) + if args.debug: + logging.root.setLevel(logging.DEBUG) + log.debug(args) args.func(args) if __name__ == "__main__": From d15d65924835d0c0f1c4d559bf44fcc9cdccb21c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:20:43 +0200 Subject: [PATCH 099/107] increment version for next release Signed-off-by: Nico Schottelius --- lib/cdist/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cdist/__init__.py b/lib/cdist/__init__.py index 192e5001..a0ca2ba2 100644 --- a/lib/cdist/__init__.py +++ b/lib/cdist/__init__.py @@ -19,7 +19,7 @@ # # -VERSION = "2.0.2" +VERSION = "2.0.3" class Error(Exception): """Base exception class for this project""" From 26c0d5d9aa28b67b1da47e326bdc3facc045d29a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:27:28 +0200 Subject: [PATCH 100/107] upcoming changes for 2.0.3 Signed-off-by: Nico Schottelius --- doc/changelog | 3 +++ lib/cdist/config.py | 8 +++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/doc/changelog b/doc/changelog index 39455351..9fb11307 100644 --- a/doc/changelog +++ b/doc/changelog @@ -1,3 +1,6 @@ +2.0.3: + * Improved logging, added --verbose, by more quiet by default + 2.0.2: 2011-09-27 * Add support for detection of OpenWall Linux (Matthias Teege) * Add support for __debug variable in manifests diff --git a/lib/cdist/config.py b/lib/cdist/config.py index f50195aa..51615c28 100644 --- a/lib/cdist/config.py +++ b/lib/cdist/config.py @@ -123,6 +123,7 @@ class Config: def run_initial_manifest(self): """Run the initial manifest""" + log.info("Running initial manifest %s", self.path.initial_manifest) env = { "__manifest" : self.path.manifest_dir } self.run_manifest(self.path.initial_manifest, extra_env=env) @@ -243,12 +244,13 @@ class Config: self.run_global_explores() self.run_initial_manifest() + log.info("Running object manifests and type explorers") + old_objects = [] objects = self.path.list_objects() # Continue process until no new objects are created anymore while old_objects != objects: - log.debug("Prepare stage") old_objects = list(objects) for cdist_object in objects: if cdist_object in self.objects_prepared: @@ -263,7 +265,7 @@ class Config: def stage_run(self): """The final (and real) step of deployment""" - log.debug("Actual run objects") + log.info("Generating and executing code") # Now do the final steps over the existing objects for cdist_object in self.path.list_objects(): log.debug("Run object: %s", cdist_object) @@ -306,7 +308,7 @@ def config(args): if args.parallel: for p in process.keys(): - log.debug("Joining %s", p) + log.debug("Joining process %s", p) process[p].join() time_end = datetime.datetime.now() From 298688eb35eef7b4faee8f538062145272b7308d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:31:39 +0200 Subject: [PATCH 101/107] hint for archlinux users Signed-off-by: Nico Schottelius --- README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README b/README index 555d28d6..28580c38 100644 --- a/README +++ b/README @@ -93,6 +93,12 @@ cdist was tested or is know to run on at least Ensure you have Python 3.x and the **argparse** module installed on the machine you use to **deploy to the targets**. +#### Archlinux + +Archlinux already has python >= 3.2, so you only need to do: + + pacman -S python + #### Debian aptitude install python3 python3-setuptools From fdb694bc42a09f94a71765c85d3fcaba74f2c70e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Sep 2011 17:36:01 +0200 Subject: [PATCH 102/107] ++todo Signed-off-by: Nico Schottelius --- doc/dev/todo/TAKEME | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/dev/todo/TAKEME b/doc/dev/todo/TAKEME index 8be1da54..4cdb6fcf 100644 --- a/doc/dev/todo/TAKEME +++ b/doc/dev/todo/TAKEME @@ -34,6 +34,8 @@ USER INTERFACE -> given after manifest run already! - use absent/present for state by default? +- buggy output with packages that don't exist in archlinux and fedora: + python3 vs. python TYPES ------ From e162861b17e3bd4287ddd55e8532bb00c49ffae1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:06:20 +0200 Subject: [PATCH 103/107] consistently support -h, -v, -d in all commands Signed-off-by: Nico Schottelius --- bin/cdist | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/bin/cdist b/bin/cdist index 7d78ff51..0bf3ed9c 100755 --- a/bin/cdist +++ b/bin/cdist @@ -39,16 +39,17 @@ def commandline(): # Construct parser others can reuse parser = {} # Options _all_ parsers have in common - parser['most'] = argparse.ArgumentParser(add_help=False) - parser['most'].add_argument('-d', '--debug', + parser['loglevel'] = argparse.ArgumentParser(add_help=False) + parser['loglevel'].add_argument('-d', '--debug', help='Set log level to debug', action='store_true', default=False) - parser['most'].add_argument('-v', '--verbose', - help='Set log level to info, is more verbose', + parser['loglevel'].add_argument('-v', '--verbose', + help='Set log level to info, be more verbose', action='store_true', default=False) # Main subcommand parser - parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION) + parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION, + parents=[parser['loglevel']]) parser['main'].add_argument('-V', '--version', help='Show version', action='version', version='%(prog)s ' + cdist.VERSION) @@ -56,7 +57,7 @@ def commandline(): # Banner parser['banner'] = parser['sub'].add_parser('banner', - add_help=False) + parents=[parser['loglevel']]) parser['banner'].set_defaults(func=cdist.banner.banner) # Config and install (common stuff) @@ -78,12 +79,12 @@ def commandline(): # Config parser['config'] = parser['sub'].add_parser('config', - parents=[parser['most'], parser['configinstall']]) + parents=[parser['loglevel'], parser['configinstall']]) parser['config'].set_defaults(func=cdist.config.config) # Install parser['install'] = parser['sub'].add_parser('install', - parents=[parser['most'], parser['configinstall']]) + parents=[parser['loglevel'], parser['configinstall']]) parser['install'].set_defaults(func=cdist.install.install) for p in parser: From 5d018393a9a25e41121d12c8697a0c7368cd1c2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:13:39 +0200 Subject: [PATCH 104/107] test if banner works Signed-off-by: Nico Schottelius --- test.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test.py b/test.py index b6a575e9..8fe759a6 100755 --- a/test.py +++ b/test.py @@ -24,6 +24,7 @@ import os import sys import shutil +import subprocess import tempfile import unittest @@ -149,5 +150,11 @@ class Config(unittest.TestCase): self.assertFalse(failed) + +class UI(unittest.TestCase): + def test_banner(self): + self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + + if __name__ == '__main__': unittest.main() From 3265755bb9197e54e66de791d24b7eff70c980e1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:15:04 +0200 Subject: [PATCH 105/107] test help function Signed-off-by: Nico Schottelius --- test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test.py b/test.py index 8fe759a6..d03ad0f1 100755 --- a/test.py +++ b/test.py @@ -34,6 +34,8 @@ sys.path.insert(0, os.path.abspath( cdist_exec_path = os.path.abspath( os.path.join(os.path.dirname(os.path.realpath(__file__)), "bin/cdist")) +cdist_commands=["banner", "config", "install"] + import cdist import cdist.config import cdist.exec @@ -155,6 +157,10 @@ class UI(unittest.TestCase): def test_banner(self): self.assertEqual(subprocess.call([cdist_exec_path, "banner"]), 0) + def test_help(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + if __name__ == '__main__': unittest.main() From f357eb3038c9d5eef63c506384011f1d0a7e0067 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Sep 2011 09:17:27 +0200 Subject: [PATCH 106/107] test config of localhost Signed-off-by: Nico Schottelius --- test.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test.py b/test.py index d03ad0f1..5d57fae4 100755 --- a/test.py +++ b/test.py @@ -161,6 +161,11 @@ class UI(unittest.TestCase): for cmd in cdist_commands: self.assertEqual(subprocess.call([cdist_exec_path, cmd, "-h"]), 0) + # FIXME: mockup needed + def test_config_localhost(self): + for cmd in cdist_commands: + self.assertEqual(subprocess.call([cdist_exec_path, "config", "localhost"]), 0) + if __name__ == '__main__': unittest.main() From ab061770f4c4fa2b5fdde1fea46a73c156776278 Mon Sep 17 00:00:00 2001 From: Daniel Roth Date: Sat, 1 Oct 2011 13:08:07 +0200 Subject: [PATCH 107/107] Installation with gentoo --- README | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/README b/README index 28580c38..aab4c3a5 100644 --- a/README +++ b/README @@ -105,6 +105,16 @@ Archlinux already has python >= 3.2, so you only need to do: easy_install3 argparse +#### Gentoo + +Gentoo only provides python 3.2 in testing packages (http://www.gentoo.org/doc/en/handbook/handbook-x86.xml?part=3&chap=3). +If you want to ensure nothing breaks you must set back the python version to what was default before. + + emerge -av =python-3.2.2 --autounmask-write + emerge -av =python-3.2.2 + eselect python list + eselect python list set python3.2 + ### Get cdist You can clone cdist from git, which gives you the advantage of having