From 136f2ecd87cb2cd09777a93a4bd44b65aa61f567 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 1 Sep 2017 14:08:50 +0200 Subject: [PATCH] Add helpers for cdist config/install integration. (#551) Implement simple integration API. --- cdist/argparse.py | 6 +- cdist/config.py | 21 +++- cdist/configuration.py | 11 +- cdist/exec/remote.py | 3 + cdist/integration.py | 153 +++++++++++++++++++++++++++ cdist/test/configuration/__init__.py | 9 +- docs/src/cdist-integration.rst | 47 ++++++++ docs/src/index.rst | 1 + 8 files changed, 238 insertions(+), 13 deletions(-) create mode 100644 cdist/integration.py create mode 100644 docs/src/cdist-integration.rst diff --git a/cdist/argparse.py b/cdist/argparse.py index f0ea802a..28d3840e 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -73,8 +73,6 @@ def check_beta(args_dict): def check_positive_int(value): - import argparse - try: val = int(value) except ValueError: @@ -416,10 +414,10 @@ def handle_loglevel(args): logging.root.setLevel(_verbosity_level[args.verbose]) -def parse_and_configure(argv): +def parse_and_configure(argv, singleton=True): parser = get_parsers() parser_args = parser['main'].parse_args(argv) - cfg = cdist.configuration.Configuration(parser_args) + cfg = cdist.configuration.Configuration(parser_args, singleton=singleton) args = cfg.get_args() # Loglevels are handled globally in here handle_loglevel(args) diff --git a/cdist/config.py b/cdist/config.py index 779365f3..0435201c 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -45,7 +45,7 @@ class Config(object): """Cdist main class to hold arbitrary data""" def __init__(self, local, remote, dry_run=False, jobs=None, - cleanup_cmds=None): + cleanup_cmds=None, remove_remote_files_dirs=False): self.local = local self.remote = remote @@ -56,6 +56,7 @@ class Config(object): self.cleanup_cmds = cleanup_cmds else: self.cleanup_cmds = [] + self.remove_remote_files_dirs = remove_remote_files_dirs self.explorer = core.Explorer(self.local.target_host, self.local, self.remote, jobs=self.jobs) @@ -67,6 +68,15 @@ class Config(object): self.local.create_files_dirs() self.remote.create_files_dirs() + def _remove_remote_files_dirs(self): + """Remove remote files and directories for the run""" + self.remote.remove_files_dirs() + + def _remove_files_dirs(self): + """Remove files and directories for the run""" + if self.remove_remote_files_dirs: + self._remove_remote_files_dirs() + @staticmethod def hosts(source): try: @@ -283,7 +293,7 @@ class Config(object): @classmethod def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, - parallel, configuration): + parallel, configuration, remove_remote_files_dirs=False): """Configure ONE system. If operating in parallel then return tuple (host, True|False, ) so that main process knows for which host function was successful. @@ -311,7 +321,8 @@ class Config(object): add_conf_dirs=args.conf_dir, cache_path_pattern=args.cache_path_pattern, quiet_mode=args.quiet, - configuration=configuration) + configuration=configuration, + exec_path=sys.argv[0]) remote = cdist.exec.remote.Remote( target_host=target_host, @@ -326,7 +337,8 @@ class Config(object): if cleanup_cmd: cleanup_cmds.append(cleanup_cmd) c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs, - cleanup_cmds=cleanup_cmds) + cleanup_cmds=cleanup_cmds, + remove_remote_files_dirs=remove_remote_files_dirs) c.run() except cdist.Error as e: @@ -367,6 +379,7 @@ class Config(object): self.manifest.run_initial_manifest(self.local.initial_manifest) self.iterate_until_finished() self.cleanup() + self._remove_files_dirs() self.local.save_cache(start_time) self.log.info("Finished successful run in {:.2f} seconds".format( diff --git a/cdist/configuration.py b/cdist/configuration.py index fa9671d0..b1154ae8 100644 --- a/cdist/configuration.py +++ b/cdist/configuration.py @@ -32,9 +32,12 @@ class Singleton(type): instance = None def __call__(cls, *args, **kwargs): - if not cls.instance: - cls.instance = super(Singleton, cls).__call__(*args, **kwargs) - return cls.instance + if 'singleton' in kwargs and kwargs['singleton'] == False: + return super(Singleton, cls).__call__(*args, **kwargs) + else: + if not cls.instance: + cls.instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls.instance _VERBOSITY_VALUES = ( @@ -294,7 +297,7 @@ class Configuration(metaclass=Singleton): return None def __init__(self, command_line_args, env=os.environ, - config_files=default_config_files): + config_files=default_config_files, singleton=True): self.command_line_args = command_line_args self.args = self._convert_args(command_line_args) self.env = env diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 237c2a9f..b834cd97 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -116,6 +116,9 @@ class Remote(object): self.run(["chmod", "0700", self.base_path]) self.mkdir(self.conf_path) + def remove_files_dirs(self): + self.rmdir(self.base_path) + def rmfile(self, path): """Remove file on the remote side.""" self.log.trace("Remote rm: %s", path) diff --git a/cdist/integration.py b/cdist/integration.py new file mode 100644 index 00000000..e2f9be6e --- /dev/null +++ b/cdist/integration.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import cdist +# needed for cdist.argparse +import cdist.banner +import cdist.config +import cdist.install +import cdist.shell +import cdist.inventory +import cdist.argparse +import cdist.log +import cdist.config +import cdist.install +import sys +import os +import os.path +import collections +import uuid + + +def find_cdist_exec_in_path(): + """Search cdist executable in os.get_exec_path() entries. + """ + for path in os.get_exec_path(): + cdist_path = os.path.join(path, 'cdist') + if os.access(cdist_path, os.X_OK): + return cdist_path + return None + + +_mydir = os.path.dirname(__file__) + + +def find_cdist_exec(): + """Search cdist executable starting from local lib directory. + + Detect if ../scripts/cdist (from local lib direcotry) exists and + if it is executable. If not then try to find cdist exec path in + os.get_exec_path() entries. If no cdist path is found rasie + cdist.Error. + """ + cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts', + 'cdist')) + if os.access(cdist_path, os.X_OK): + return cdist_path + cdist_path = find_cdist_exec_in_path() + if not cdist_path: + raise cdist.Error('Cannot find cdist executable from local lib ' + 'directory: {}, nor in PATH: {}.'.format( + _mydir, os.environ.get('PATH'))) + return cdist_path + + +ACTION_CONFIG = 'config' +ACTION_INSTALL = 'install' + + +def _process_hosts_simple(action, host, manifest, verbose, + cdist_path=None): + """Perform cdist action ('config' or 'install') on hosts with specified + manifest using default other cdist options. host parameter can be a + string or iterbale of hosts. verbose is a desired verbosity level + which defaults to VERBOSE_INFO. cdist_path is path to cdist executable, + if it is None then integration lib tries to find it. + """ + if isinstance(host, str): + hosts = [host, ] + elif isinstance(host, collections.Iterable): + hosts = host + else: + raise cdist.Error('Invalid host argument: {}'.format(host)) + + # Setup sys.argv[0] since cdist relies on command line invocation. + if not cdist_path: + cdist_path = find_cdist_exec() + sys.argv[0] = cdist_path + + cname = action.title() + module = getattr(cdist, action) + theclass = getattr(module, cname) + + # Build argv for cdist and use argparse for argument parsing. + remote_out_dir_base = os.path.join('/', 'var', 'lib', 'cdist') + uid = str(uuid.uuid1()) + out_dir = remote_out_dir_base + uid + cache_path_pattern = '%h-' + uid + argv = [action, '-i', manifest, '-r', out_dir, '-C', cache_path_pattern, ] + for i in range(verbose): + argv.append('-v') + for x in hosts: + argv.append(x) + + parser, cfg = cdist.argparse.parse_and_configure(argv, singleton=False) + args = cfg.get_args() + configuration = cfg.get_config(section='GLOBAL') + + theclass.construct_remote_exec_copy_patterns(args) + base_root_path = theclass.create_base_root_path(None) + + for target_host in args.host: + host_base_path, hostdir = theclass.create_host_base_dirs( + target_host, base_root_path) + theclass.onehost(target_host, None, host_base_path, hostdir, args, + parallel=False, configuration=configuration, + remove_remote_files_dirs=True) + + +def configure_hosts_simple(host, manifest, + verbose=cdist.argparse.VERBOSE_INFO, + cdist_path=None): + """Configure hosts with specified manifest using default other cdist + options. host parameter can be a string or iterbale of hosts. verbose + is a desired verbosity level which defaults to VERBOSE_INFO. + cdist_path is path to cdist executable, if it is None then integration + lib tries to find it. + """ + _process_hosts_simple(action=ACTION_CONFIG, host=host, + manifest=manifest, verbose=verbose, + cdist_path=cdist_path) + + +def install_hosts_simple(host, manifest, + verbose=cdist.argparse.VERBOSE_INFO, + cdist_path=None): + """Install hosts with specified manifest using default other cdist + options. host parameter can be a string or iterbale of hosts. verbose + is a desired verbosity level which defaults to VERBOSE_INFO. + cdist_path is path to cdist executable, if it is None then integration + lib tries to find it. + """ + _process_hosts_simple(action=ACTION_INSTALL, host=host, + manifest=manifest, verbose=verbose, + cdist_path=cdist_path) diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py index 2ffd8255..a91e8e8b 100644 --- a/cdist/test/configuration/__init__.py +++ b/cdist/test/configuration/__init__.py @@ -236,9 +236,16 @@ class ConfigurationTestCase(test.CdistTestCase): x = cc.Configuration(None) args = argparse.Namespace() args.a = 'a' - y = cc.Configuration() + y = cc.Configuration(args) self.assertIs(x, y) + def test_non_singleton(self): + x = cc.Configuration(None, singleton=False) + args = argparse.Namespace() + args.a = 'a' + y = cc.Configuration(args, singleton=False) + self.assertIsNot(x, y) + def test_read_config_file(self): config = cc.Configuration(None, env={}, config_files=()) d = config._read_config_file(self.config_file) diff --git a/docs/src/cdist-integration.rst b/docs/src/cdist-integration.rst new file mode 100644 index 00000000..13880cd3 --- /dev/null +++ b/docs/src/cdist-integration.rst @@ -0,0 +1,47 @@ +cdist integration / using cdist as library +========================================== + +Description +----------- + +cdist can be integrate with other applications by importing cdist and other +cdist modules and setting all by hand. There are also helper functions which +aim to ease this integration. Just import **cdist.integration** and use its +functions: + +* :strong:`cdist.integration.configure_hosts_simple` for configuration +* :strong:`cdist.integration.install_hosts_simple` for installation. + +Functions require `host` and `manifest` parameters. +`host` can be specified as a string representing host or as iterable +of hosts. `manifest` is a path to initial manifest. For other cdist +options default values will be used. `verbose` is a desired verbosity +level which defaults to VERBOSE_INFO. `cdist_path` parameter specifies +path to cdist executable, if it is `None` then functions will try to +find it first from local lib directory and then in PATH. + +In case of cdist error :strong:`cdist.Error` exception is raised. + +:strong:`WARNING`: cdist integration helper functions are not yet stable! + +Examples +-------- + +.. code-block:: sh + + # configure host from python interactive shell + >>> import cdist.integration + >>> cdist.integration.configure_hosts_simple('185.203.114.185', + ... '~/.cdist/manifest/init') + + # configure host from python interactive shell, specifying verbosity level + >>> import cdist.integration + >>> cdist.integration.configure_hosts_simple( + ... '185.203.114.185', '~/.cdist/manifest/init', + ... verbose=cdist.argparse.VERBOSE_TRACE) + + # configure specified dns hosts from python interactive shell + >>> import cdist.integration + >>> hosts = ('dns1.ungleich.ch', 'dns2.ungleich.ch', 'dns3.ungleich.ch', ) + >>> cdist.integration.configure_hosts_simple(hosts, + ... '~/.cdist/manifest/init') diff --git a/docs/src/index.rst b/docs/src/index.rst index 652c6082..d662fd73 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -26,6 +26,7 @@ Contents: cdist-messaging cdist-parallelization cdist-inventory + cdist-integration cdist-reference cdist-best-practice cdist-stages