Add helpers for cdist config/install integration. (#551)

Implement simple integration API.

cdist/argparse.py

@@ -73,8 +73,6 @@ def check_beta(args_dict):
 
 
 def check_positive_int(value):
-    import argparse
-
     try:
         val = int(value)
     except ValueError:
@@ -416,10 +414,10 @@ def handle_loglevel(args):
     logging.root.setLevel(_verbosity_level[args.verbose])
 
 
-def parse_and_configure(argv):
+def parse_and_configure(argv, singleton=True):
     parser = get_parsers()
     parser_args = parser['main'].parse_args(argv)
-    cfg = cdist.configuration.Configuration(parser_args)
+    cfg = cdist.configuration.Configuration(parser_args, singleton=singleton)
     args = cfg.get_args()
     # Loglevels are handled globally in here
     handle_loglevel(args)

cdist/config.py

@@ -45,7 +45,7 @@ class Config(object):
     """Cdist main class to hold arbitrary data"""
 
     def __init__(self, local, remote, dry_run=False, jobs=None,
-                 cleanup_cmds=None):
+                 cleanup_cmds=None, remove_remote_files_dirs=False):
 
         self.local = local
         self.remote = remote
@@ -56,6 +56,7 @@ class Config(object):
             self.cleanup_cmds = cleanup_cmds
         else:
             self.cleanup_cmds = []
+        self.remove_remote_files_dirs = remove_remote_files_dirs
 
         self.explorer = core.Explorer(self.local.target_host, self.local,
                                       self.remote, jobs=self.jobs)
@@ -67,6 +68,15 @@ class Config(object):
         self.local.create_files_dirs()
         self.remote.create_files_dirs()
 
+    def _remove_remote_files_dirs(self):
+        """Remove remote files and directories for the run"""
+        self.remote.remove_files_dirs()
+
+    def _remove_files_dirs(self):
+        """Remove files and directories for the run"""
+        if self.remove_remote_files_dirs:
+            self._remove_remote_files_dirs()
+
     @staticmethod
     def hosts(source):
         try:
@@ -283,7 +293,7 @@ class Config(object):
 
     @classmethod
     def onehost(cls, host, host_tags, host_base_path, host_dir_name, args,
-                parallel, configuration):
+                parallel, configuration, remove_remote_files_dirs=False):
         """Configure ONE system.
            If operating in parallel then return tuple (host, True|False, )
            so that main process knows for which host function was successful.
@@ -311,7 +321,8 @@ class Config(object):
                 add_conf_dirs=args.conf_dir,
                 cache_path_pattern=args.cache_path_pattern,
                 quiet_mode=args.quiet,
-                configuration=configuration)
+                configuration=configuration,
+                exec_path=sys.argv[0])
 
             remote = cdist.exec.remote.Remote(
                 target_host=target_host,
@@ -326,7 +337,8 @@ class Config(object):
             if cleanup_cmd:
                 cleanup_cmds.append(cleanup_cmd)
             c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs,
-                    cleanup_cmds=cleanup_cmds)
+                    cleanup_cmds=cleanup_cmds,
+                    remove_remote_files_dirs=remove_remote_files_dirs)
             c.run()
 
         except cdist.Error as e:
@@ -367,6 +379,7 @@ class Config(object):
         self.manifest.run_initial_manifest(self.local.initial_manifest)
         self.iterate_until_finished()
         self.cleanup()
+        self._remove_files_dirs()
 
         self.local.save_cache(start_time)
         self.log.info("Finished successful run in {:.2f} seconds".format(

cdist/configuration.py

@@ -32,9 +32,12 @@ class Singleton(type):
     instance = None
 
     def __call__(cls, *args, **kwargs):
-        if not cls.instance:
+        if 'singleton' in kwargs and kwargs['singleton'] == False:
-            cls.instance = super(Singleton, cls).__call__(*args, **kwargs)
+            return super(Singleton, cls).__call__(*args, **kwargs)
-        return cls.instance
+        else:
+            if not cls.instance:
+                cls.instance = super(Singleton, cls).__call__(*args, **kwargs)
+            return cls.instance
 
 
 _VERBOSITY_VALUES = (
@@ -294,7 +297,7 @@ class Configuration(metaclass=Singleton):
             return None
 
     def __init__(self, command_line_args, env=os.environ,
-                 config_files=default_config_files):
+                 config_files=default_config_files, singleton=True):
         self.command_line_args = command_line_args
         self.args = self._convert_args(command_line_args)
         self.env = env

cdist/exec/remote.py

@@ -116,6 +116,9 @@ class Remote(object):
         self.run(["chmod", "0700", self.base_path])
         self.mkdir(self.conf_path)
 
+    def remove_files_dirs(self):
+        self.rmdir(self.base_path)
+
     def rmfile(self, path):
         """Remove file on the remote side."""
         self.log.trace("Remote rm: %s", path)

cdist/integration.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import cdist
+# needed for cdist.argparse
+import cdist.banner
+import cdist.config
+import cdist.install
+import cdist.shell
+import cdist.inventory
+import cdist.argparse
+import cdist.log
+import cdist.config
+import cdist.install
+import sys
+import os
+import os.path
+import collections
+import uuid
+
+
+def find_cdist_exec_in_path():
+    """Search cdist executable in os.get_exec_path() entries.
+    """
+    for path in os.get_exec_path():
+        cdist_path = os.path.join(path, 'cdist')
+        if os.access(cdist_path, os.X_OK):
+            return cdist_path
+    return None
+
+
+_mydir = os.path.dirname(__file__)
+
+
+def find_cdist_exec():
+    """Search cdist executable starting from local lib directory.
+
+    Detect if ../scripts/cdist (from local lib direcotry) exists and
+    if it is executable. If not then try to find cdist exec path in
+    os.get_exec_path() entries. If no cdist path is found rasie
+    cdist.Error.
+    """
+    cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts',
+                                              'cdist'))
+    if os.access(cdist_path, os.X_OK):
+        return cdist_path
+    cdist_path = find_cdist_exec_in_path()
+    if not cdist_path:
+        raise cdist.Error('Cannot find cdist executable from local lib '
+                          'directory: {}, nor in PATH: {}.'.format(
+                              _mydir, os.environ.get('PATH')))
+    return cdist_path
+
+
+ACTION_CONFIG = 'config'
+ACTION_INSTALL = 'install'
+
+
+def _process_hosts_simple(action, host, manifest, verbose,
+                          cdist_path=None):
+    """Perform cdist action ('config' or 'install') on hosts with specified
+       manifest using default other cdist options. host parameter can be a
+       string or iterbale of hosts. verbose is a desired verbosity level
+       which defaults to VERBOSE_INFO. cdist_path is path to cdist executable,
+       if it is None then integration lib tries to find it.
+    """
+    if isinstance(host, str):
+        hosts = [host, ]
+    elif isinstance(host, collections.Iterable):
+        hosts = host
+    else:
+        raise cdist.Error('Invalid host argument: {}'.format(host))
+
+    # Setup sys.argv[0] since cdist relies on command line invocation.
+    if not cdist_path:
+        cdist_path = find_cdist_exec()
+    sys.argv[0] = cdist_path
+
+    cname = action.title()
+    module = getattr(cdist, action)
+    theclass = getattr(module, cname)
+
+    # Build argv for cdist and use argparse for argument parsing.
+    remote_out_dir_base = os.path.join('/', 'var', 'lib', 'cdist')
+    uid = str(uuid.uuid1())
+    out_dir = remote_out_dir_base + uid
+    cache_path_pattern = '%h-' + uid
+    argv = [action, '-i', manifest, '-r', out_dir, '-C', cache_path_pattern, ]
+    for i in range(verbose):
+        argv.append('-v')
+    for x in hosts:
+        argv.append(x)
+
+    parser, cfg = cdist.argparse.parse_and_configure(argv, singleton=False)
+    args = cfg.get_args()
+    configuration = cfg.get_config(section='GLOBAL')
+
+    theclass.construct_remote_exec_copy_patterns(args)
+    base_root_path = theclass.create_base_root_path(None)
+
+    for target_host in args.host:
+        host_base_path, hostdir = theclass.create_host_base_dirs(
+            target_host, base_root_path)
+        theclass.onehost(target_host, None, host_base_path, hostdir, args,
+                         parallel=False, configuration=configuration,
+                         remove_remote_files_dirs=True)
+
+
+def configure_hosts_simple(host, manifest,
+                           verbose=cdist.argparse.VERBOSE_INFO,
+                           cdist_path=None):
+    """Configure hosts with specified manifest using default other cdist
+       options. host parameter can be a string or iterbale of hosts. verbose
+       is a desired verbosity level which defaults to VERBOSE_INFO.
+       cdist_path is path to cdist executable, if it is None then integration
+       lib tries to find it.
+    """
+    _process_hosts_simple(action=ACTION_CONFIG, host=host,
+                          manifest=manifest, verbose=verbose,
+                          cdist_path=cdist_path)
+
+
+def install_hosts_simple(host, manifest,
+                         verbose=cdist.argparse.VERBOSE_INFO,
+                         cdist_path=None):
+    """Install hosts with specified manifest using default other cdist
+       options. host parameter can be a string or iterbale of hosts. verbose
+       is a desired verbosity level which defaults to VERBOSE_INFO.
+       cdist_path is path to cdist executable, if it is None then integration
+       lib tries to find it.
+    """
+    _process_hosts_simple(action=ACTION_INSTALL, host=host,
+                          manifest=manifest, verbose=verbose,
+                          cdist_path=cdist_path)

cdist/test/configuration/__init__.py

@@ -236,9 +236,16 @@ class ConfigurationTestCase(test.CdistTestCase):
         x = cc.Configuration(None)
         args = argparse.Namespace()
         args.a = 'a'
-        y = cc.Configuration()
+        y = cc.Configuration(args)
         self.assertIs(x, y)
 
+    def test_non_singleton(self):
+        x = cc.Configuration(None, singleton=False)
+        args = argparse.Namespace()
+        args.a = 'a'
+        y = cc.Configuration(args, singleton=False)
+        self.assertIsNot(x, y)
+
     def test_read_config_file(self):
         config = cc.Configuration(None, env={}, config_files=())
         d = config._read_config_file(self.config_file)

docs/src/cdist-integration.rst

@@ -0,0 +1,47 @@
+cdist integration / using cdist as library
+==========================================
+
+Description
+-----------
+
+cdist can be integrate with other applications by importing cdist and other
+cdist modules and setting all by hand. There are also helper functions which
+aim to ease this integration. Just import **cdist.integration** and use its
+functions:
+
+* :strong:`cdist.integration.configure_hosts_simple` for configuration
+* :strong:`cdist.integration.install_hosts_simple` for installation.
+
+Functions require `host` and `manifest` parameters.
+`host` can be specified as a string representing host or as iterable
+of hosts. `manifest` is a path to initial manifest. For other cdist
+options default values will be used. `verbose` is a desired verbosity
+level which defaults to VERBOSE_INFO. `cdist_path` parameter specifies
+path to cdist executable, if it is `None` then functions will try to
+find it first from local lib directory and then in PATH.
+
+In case of cdist error :strong:`cdist.Error` exception is raised.
+
+:strong:`WARNING`: cdist integration helper functions are not yet stable!
+
+Examples
+--------
+
+.. code-block:: sh
+
+    # configure host from python interactive shell
+    >>> import cdist.integration
+    >>> cdist.integration.configure_hosts_simple('185.203.114.185',
+    ...                                          '~/.cdist/manifest/init')
+
+    # configure host from python interactive shell, specifying verbosity level
+    >>> import cdist.integration
+    >>> cdist.integration.configure_hosts_simple(
+    ...     '185.203.114.185', '~/.cdist/manifest/init',
+    ...     verbose=cdist.argparse.VERBOSE_TRACE)
+
+    # configure specified dns hosts from python interactive shell
+    >>> import cdist.integration
+    >>> hosts = ('dns1.ungleich.ch', 'dns2.ungleich.ch', 'dns3.ungleich.ch', )
+    >>> cdist.integration.configure_hosts_simple(hosts,
+    ...                                          '~/.cdist/manifest/init')

docs/src/index.rst

@@ -26,6 +26,7 @@ Contents:
    cdist-messaging
    cdist-parallelization
    cdist-inventory
+   cdist-integration
    cdist-reference
    cdist-best-practice
    cdist-stages