From 0a85d913cc4497606fbbad99e5db8222b2f70799 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 27 Jul 2017 09:49:06 +0200 Subject: [PATCH] Optimize dir transfer. --- cdist/exec/remote.py | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 7a1fab3d..8b10b324 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -139,12 +139,16 @@ class Remote(object): _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) - def _transfer_dir_sequential(self, source, destination): + def _transfer_dir_commands(self, source, destination): for f in glob.glob1(source, '*'): command = self._copy.split() path = os.path.join(source, f) command.extend([path, '{0}:{1}'.format( _wrap_addr(self.target_host[0]), destination)]) + yield command + + def _transfer_dir_sequential(self, source, destination): + for command in self._transfer_dir_commands: self._run_command(command) def _transfer_dir_parallel(self, source, destination, jobs): @@ -155,14 +159,12 @@ class Remote(object): multiprocessing.get_start_method())) self.log.trace(("Starting multiprocessing Pool for parallel " "remote transfer")) - args = [] - for f in glob.glob1(source, '*'): - command = self._copy.split() - path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - args.append((command, )) - mp_pool_run(self._run_command, args, jobs=jobs) + args = [(command, ) for command in self._transfer_dir_commands] + if len(args) == 1: + self.log.debug("Only one dir entry, transfering sequentially") + self._run_command(args[0]) + else: + mp_pool_run(self._run_command, args, jobs=jobs) self.log.trace(("Multiprocessing for parallel transfer " "finished"))