fancsali/cdist-backup
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Create mp_pool_run helper function for running in parallel.

Browse source
This commit is contained in:
Darko Poljak 2016-12-08 21:48:59 +01:00
commit e5a6599ccb
4 changed files with 79 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

30
cdist/exec/remote.py
View file

@ -31,6 +31,7 @@ import multiprocessing
import cdist
import cdist.exec.util as exec_util
import cdist.util.ipaddr as ipaddr
from cdist.mputil import mp_pool_run
def _wrap_addr(addr):
@ -152,25 +153,16 @@ class Remote(object):
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for parallel "
"remote transfer"))
with multiprocessing.Pool(jobs) as pool:
self.log.debug("Starting async for parallel transfer")
commands = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
commands.append(command)
results = [
pool.apply_async(self._run_command, (cmd,))
for cmd in commands
]
self.log.debug("Waiting async results for parallel transfer")
for r in results:
r.get() # self._run_command returns None
self.log.debug(("Multiprocessing for parallel transfer "
"finished"))
args = []
for f in glob.glob1(source, '*'):
command = self._copy.split()
path = os.path.join(source, f)
command.extend([path, '{0}:{1}'.format(
_wrap_addr(self.target_host[0]), destination)])
args.append((command, ))
mp_pool_run(self._run_command, args, jobs=jobs)
self.log.debug(("Multiprocessing for parallel transfer "
"finished"))
def run_script(self, script, env=None, return_output=False):
"""Run the given script with the given environment on the remote side.