Merge remote-tracking branch 'ungleich/master' into keyboard-locale_system-types

cdist/__init__.py

@@ -21,6 +21,7 @@
 
 import os
 import subprocess
+import hashlib
 
 import cdist.version
 
@@ -82,3 +83,11 @@ def file_to_list(filename):
         lines = []
 
     return lines
+
+
+def str_hash(s):
+    """Return hash of string s"""
+    if isinstance(s, str):
+        return hashlib.md5(s.encode('utf-8')).hexdigest()
+    else:
+        raise Error("Param should be string")

cdist/config.py

@@ -27,6 +27,7 @@ import sys
 import time
 import pprint
 import itertools
+import tempfile
 
 import cdist
 
@@ -36,6 +37,36 @@ import cdist.exec.remote
 from cdist import core
 
 
+def inspect_ssh_mux_opts():
+    """Inspect whether or not ssh supports multiplexing options.
+
+       Return string containing multiplexing options if supported.
+       If ControlPath is supported then placeholder for that path is
+       specified and can be used for final string formatting.
+       For example, this function can return string:
+       "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}".
+       Then it can be formatted:
+       mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path').
+    """
+    import subprocess
+
+    wanted_mux_opts = {
+        "ControlPath": "{}",
+        "ControlMaster": "auto",
+        "ControlPersist": "125",
+    }
+    mux_opts = " ".join([" -o {}={}".format(
+        x, wanted_mux_opts[x]) for x in wanted_mux_opts])
+    try:
+        subprocess.check_output("ssh {}".format(mux_opts),
+                                stderr=subprocess.STDOUT, shell=True)
+    except subprocess.CalledProcessError as e:
+        subproc_output = e.output.decode().lower()
+        if "bad configuration option" in subproc_output:
+            return ""
+    return mux_opts
+
+
 class Config(object):
     """Cdist main class to hold arbitrary data"""
 
@@ -94,7 +125,6 @@ class Config(object):
         initial_manifest_tempfile = None
         if args.manifest == '-':
             # read initial manifest from stdin
-            import tempfile
             try:
                 handle, initial_manifest_temp_path = tempfile.mkstemp(
                         prefix='cdist.stdin.')
@@ -112,18 +142,47 @@ class Config(object):
         failed_hosts = []
         time_start = time.time()
 
+        # default remote cmd patterns
+        args.remote_exec_pattern = None
+        args.remote_copy_pattern = None
+
+        args_dict = vars(args)
+        # if remote-exec and/or remote-copy args are None then user
+        # didn't specify command line options nor env vars:
+        # inspect multiplexing options for default cdist.REMOTE_COPY/EXEC
+        if (args_dict['remote_copy'] is None or
+                args_dict['remote_exec'] is None):
+            mux_opts = inspect_ssh_mux_opts()
+            if args_dict['remote_exec'] is None:
+                args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts
+            if args_dict['remote_copy'] is None:
+                args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts
+
+        if args.out_path:
+            base_root_path = args.out_path
+        else:
+            base_root_path = tempfile.mkdtemp()
+
         hostcnt = 0
         for host in itertools.chain(cls.hosts(args.host),
                                     cls.hosts(args.hostfile)):
+            hostdir = cdist.str_hash(host)
+            host_base_path = os.path.join(base_root_path, hostdir)
+
+            log.debug("Base root path for target host \"{}\" is \"{}\"".format(
+                host, host_base_path))
+
             hostcnt += 1
             if args.parallel:
                 log.debug("Creating child process for %s", host)
                 process[host] = multiprocessing.Process(
-                        target=cls.onehost, args=(host, args, True))
+                        target=cls.onehost,
+                        args=(host, host_base_path, hostdir, args, True))
                 process[host].start()
             else:
                 try:
-                    cls.onehost(host, args, parallel=False)
+                    cls.onehost(host, host_base_path, hostdir,
+                                args, parallel=False)
                 except cdist.Error as e:
                     failed_hosts.append(host)
 
@@ -145,22 +204,42 @@ class Config(object):
                               " ".join(failed_hosts))
 
     @classmethod
-    def onehost(cls, host, args, parallel):
+    def onehost(cls, host, host_base_path, host_dir_name, args, parallel):
         """Configure ONE system"""
 
         log = logging.getLogger(host)
 
         try:
+            control_path = os.path.join(host_base_path, "ssh-control-path")
+            # If we constructed patterns for remote commands then there is
+            # placeholder for ssh ControlPath, format it and we have unique
+            # ControlPath for each host.
+            #
+            # If not then use args.remote_exec/copy that user specified.
+            if args.remote_exec_pattern:
+                remote_exec = args.remote_exec_pattern.format(control_path)
+            else:
+                remote_exec = args.remote_exec
+            if args.remote_copy_pattern:
+                remote_copy = args.remote_copy_pattern.format(control_path)
+            else:
+                remote_copy = args.remote_copy
+            log.debug("remote_exec for host \"{}\": {}".format(
+                host, remote_exec))
+            log.debug("remote_copy for host \"{}\": {}".format(
+                host, remote_copy))
+
             local = cdist.exec.local.Local(
                 target_host=host,
+                base_root_path=host_base_path,
+                host_dir_name=host_dir_name,
                 initial_manifest=args.manifest,
-                base_path=args.out_path,
                 add_conf_dirs=args.conf_dir)
 
             remote = cdist.exec.remote.Remote(
                 target_host=host,
-                remote_exec=args.remote_exec,
-                remote_copy=args.remote_copy)
+                remote_exec=remote_exec,
+                remote_copy=remote_copy)
 
             c = cls(local, remote, dry_run=args.dry_run)
             c.run()

cdist/exec/local.py

@@ -29,7 +29,6 @@ import subprocess
 import shutil
 import logging
 import tempfile
-import hashlib
 
 import cdist
 import cdist.message
@@ -48,40 +47,25 @@ class Local(object):
     """
     def __init__(self,
                  target_host,
+                 base_root_path,
+                 host_dir_name,
                  exec_path=sys.argv[0],
                  initial_manifest=None,
-                 base_path=None,
                  add_conf_dirs=None):
 
         self.target_host = target_host
-        self._init_log()
-
-        # FIXME: stopped: create base that does not require moving later
-        if base_path:
-            base_path_parent = base_path
-        else:
-            base_path_parent = tempfile.mkdtemp()
-            # TODO: the below atexit hook nukes any debug info we would have
-            #  if cdist exits with error.
-            # import atexit
-            # atexit.register(lambda: shutil.rmtree(base_path_parent))
-        self.hostdir = self._hostdir()
-        self.log.info("Calculated temp dir for target \"{}\" is "
-                      "\"{}\"".format(self.target_host, self.hostdir))
-        self.base_path = os.path.join(base_path_parent, self.hostdir)
-
-        self._init_permissions()
-
-        self.mkdir(self.base_path)
-
-        # FIXME: as well
-        self._init_cache_dir(None)
+        self.hostdir = host_dir_name
+        self.base_path = os.path.join(base_root_path, "data")
 
         self.exec_path = exec_path
         self.custom_initial_manifest = initial_manifest
-
         self._add_conf_dirs = add_conf_dirs
 
+        self._init_log()
+        self._init_permissions()
+        self.mkdir(self.base_path)
+        # FIXME: create dir that does not require moving later
+        self._init_cache_dir(None)
         self._init_paths()
         self._init_object_marker()
         self._init_conf_dirs()
@@ -98,12 +82,6 @@ class Local(object):
         else:
             return None
 
-    def _hostdir(self):
-        # Do not assume target_host is anything that can be used as a
-        # directory name.
-        # Instead use a hash, which is known to work as directory name.
-        return hashlib.md5(self.target_host.encode('utf-8')).hexdigest()
-
     def _init_log(self):
         self.log = logging.getLogger(self.target_host)
 

cdist/test/code/__init__.py

@@ -41,10 +41,13 @@ class CodeTestCase(test.CdistTestCase):
 
     def setUp(self):
         self.local_dir = self.mkdtemp()
+        self.hostdir = cdist.str_hash(self.target_host)
+        self.host_base_path = os.path.join(self.local_dir, self.hostdir)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=self.local_dir,
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir,
             exec_path=cdist.test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()

cdist/test/config/__init__.py

@@ -55,10 +55,13 @@ class ConfigRunTestCase(test.CdistTestCase):
         self.temp_dir = self.mkdtemp()
 
         self.local_dir = os.path.join(self.temp_dir, "local")
-        os.mkdir(self.local_dir)
+        self.hostdir = cdist.str_hash(self.target_host)
+        self.host_base_path = os.path.join(self.local_dir, self.hostdir)
+        os.makedirs(self.host_base_path)
         self.local = cdist.exec.local.Local(
             target_host=self.target_host,
-            base_path=self.local_dir)
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir)
 
         # Setup test objects
         self.object_base_path = op.join(self.temp_dir, 'object')
@@ -161,7 +164,8 @@ class ConfigRunTestCase(test.CdistTestCase):
         """Test if the dryrun option is working like expected"""
         drylocal = cdist.exec.local.Local(
             target_host=self.target_host,
-            base_path=self.local_dir,
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir,
             # exec_path can not derivated from sys.argv in case of unittest
             exec_path=os.path.abspath(os.path.join(
                 my_dir, '../../../scripts/cdist')),
@@ -184,3 +188,8 @@ class ConfigRunTestCase(test.CdistTestCase):
 #        first.requirements = ['__singleton_test/foo']
 #        with self.assertRaises(cdist.core.?????):
 #            self.config.iterate_until_finished()
+
+if __name__ == "__main__":
+    import unittest
+
+    unittest.main()

cdist/test/emulator/__init__.py

@@ -48,10 +48,13 @@ class EmulatorTestCase(test.CdistTestCase):
         handle, self.script = self.mkstemp(dir=self.temp_dir)
         os.close(handle)
         base_path = self.temp_dir
+        hostdir = cdist.str_hash(self.target_host)
+        host_base_path = os.path.join(base_path, hostdir)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=base_path,
+            base_root_path=host_base_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -148,10 +151,13 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase):
         handle, self.script = self.mkstemp(dir=self.temp_dir)
         os.close(handle)
         base_path = self.temp_dir
+        hostdir = cdist.str_hash(self.target_host)
+        host_base_path = os.path.join(base_path, hostdir)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=base_path,
+            base_root_path=host_base_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -235,10 +241,13 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase):
     def setUp(self):
         self.temp_dir = self.mkdtemp()
         base_path = os.path.join(self.temp_dir, "out")
+        hostdir = cdist.str_hash(self.target_host)
+        host_base_path = os.path.join(base_path, hostdir)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=base_path,
+            base_root_path=host_base_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -265,10 +274,13 @@ class OverrideTestCase(test.CdistTestCase):
         handle, self.script = self.mkstemp(dir=self.temp_dir)
         os.close(handle)
         base_path = self.temp_dir
+        hostdir = cdist.str_hash(self.target_host)
+        host_base_path = os.path.join(base_path, hostdir)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=base_path,
+            base_root_path=host_base_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -303,12 +315,15 @@ class ArgumentsTestCase(test.CdistTestCase):
     def setUp(self):
         self.temp_dir = self.mkdtemp()
         base_path = self.temp_dir
+        hostdir = cdist.str_hash(self.target_host)
+        host_base_path = os.path.join(base_path, hostdir)
         handle, self.script = self.mkstemp(dir=self.temp_dir)
         os.close(handle)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=base_path,
+            base_root_path=host_base_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()
@@ -425,10 +440,13 @@ class StdinTestCase(test.CdistTestCase):
 
         self.temp_dir = self.mkdtemp()
         base_path = os.path.join(self.temp_dir, "out")
+        hostdir = cdist.str_hash(self.target_host)
+        host_base_path = os.path.join(base_path, hostdir)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=base_path,
+            base_root_path=host_base_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
 

cdist/test/exec/local.py

@@ -47,11 +47,14 @@ class LocalTestCase(test.CdistTestCase):
         target_host = 'localhost'
         self.temp_dir = self.mkdtemp()
         self.out_parent_path = self.temp_dir
-        self.out_path = op.join(self.out_parent_path, target_host)
+        self.hostdir = cdist.str_hash(target_host)
+        self.host_base_path = op.join(self.out_parent_path, self.hostdir)
+        self.out_path = op.join(self.host_base_path, "data")
 
         self.local = local.Local(
             target_host=target_host,
-            base_path=self.out_parent_path,
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir,
             exec_path=test.cdist_exec_path
         )
 
@@ -109,7 +112,8 @@ class LocalTestCase(test.CdistTestCase):
 
         link_test_local = local.Local(
             target_host='localhost',
-            base_path=self.out_parent_path,
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir,
             exec_path=test.cdist_exec_path,
         )
 
@@ -127,7 +131,8 @@ class LocalTestCase(test.CdistTestCase):
 
         link_test_local = local.Local(
             target_host='localhost',
-            base_path=self.out_parent_path,
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir]
         )
@@ -148,7 +153,8 @@ class LocalTestCase(test.CdistTestCase):
 
         link_test_local = local.Local(
             target_host='localhost',
-            base_path=self.out_parent_path,
+            base_root_path=self.host_base_path,
+            host_dir_name=self.hostdir,
             exec_path=test.cdist_exec_path,
         )
 

cdist/test/explorer/__init__.py

@@ -42,12 +42,15 @@ class ExplorerClassTestCase(test.CdistTestCase):
     def setUp(self):
         self.temp_dir = self.mkdtemp()
         self.local_path = os.path.join(self.temp_dir, "local")
+        hostdir = cdist.str_hash(self.target_host)
+        base_root_path = os.path.join(self.local_path, hostdir)
         self.remote_base_path = os.path.join(self.temp_dir, "remote")
         os.makedirs(self.remote_base_path)
 
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=self.local_path,
+            base_root_path=base_root_path,
+            host_dir_name=hostdir,
             exec_path=test.cdist_exec_path,
             add_conf_dirs=[conf_dir],
             )

cdist/test/manifest/__init__.py

@@ -49,9 +49,12 @@ class ManifestTestCase(test.CdistTestCase):
         self.temp_dir = self.mkdtemp()
 
         out_path = self.temp_dir
+        hostdir = cdist.str_hash(self.target_host)
+        base_root_path = os.path.join(out_path, hostdir)
         self.local = local.Local(
             target_host=self.target_host,
-            base_path=out_path,
+            base_root_path=base_root_path,
+            host_dir_name=hostdir,
             exec_path=cdist.test.cdist_exec_path,
             add_conf_dirs=[conf_dir])
         self.local.create_files_dirs()