tom/cdist
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Optimize dir transfer.

Browse source
This commit is contained in:
Darko Poljak 2017-07-27 09:49:06 +02:00
commit 0a85d913cc
1 changed files with 11 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

18
cdist/exec/remote.py
View file

@ -139,12 +139,16 @@ class Remote(object):
_wrap_addr(self.target_host[0]), destination)]) _wrap_addr(self.target_host[0]), destination)])
self._run_command(command) self._run_command(command)
def _transfer_dir_sequential(self, source, destination): def _transfer_dir_commands(self, source, destination):
for f in glob.glob1(source, '*'): for f in glob.glob1(source, '*'):
command = self._copy.split() command = self._copy.split()
path = os.path.join(source, f) path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format( command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)]) _wrap_addr(self.target_host[0]), destination)])
yield command
def _transfer_dir_sequential(self, source, destination):
for command in self._transfer_dir_commands:
self._run_command(command) self._run_command(command)
def _transfer_dir_parallel(self, source, destination, jobs): def _transfer_dir_parallel(self, source, destination, jobs):
@ -155,13 +159,11 @@ class Remote(object):
multiprocessing.get_start_method())) multiprocessing.get_start_method()))
self.log.trace(("Starting multiprocessing Pool for parallel " self.log.trace(("Starting multiprocessing Pool for parallel "
"remote transfer")) "remote transfer"))
args = [] args = [(command, ) for command in self._transfer_dir_commands]
for f in glob.glob1(source, '*'): if len(args) == 1:
command = self._copy.split() self.log.debug("Only one dir entry, transfering sequentially")
path = os.path.join(source, f) self._run_command(args[0])
command.extend([path, '{0}:{1}'.format( else:
_wrap_addr(self.target_host[0]), destination)])
args.append((command, ))
mp_pool_run(self._run_command, args, jobs=jobs) mp_pool_run(self._run_command, args, jobs=jobs)
self.log.trace(("Multiprocessing for parallel transfer " self.log.trace(("Multiprocessing for parallel transfer "
"finished")) "finished"))