marcoduif/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests 1 Projects Releases Wiki Activity

Create mp_pool_run helper function for running in parallel.

Browse source
This commit is contained in:
Darko Poljak 2016-12-08 21:48:59 +01:00
commit e5a6599ccb
4 changed files with 79 additions and 60 deletions
Download patch file Download diff file Expand all files Collapse all files

21
cdist/core/explorer.py
View file

@ -24,8 +24,7 @@ import logging
import os
import glob
import multiprocessing
import cdist
from cdist.mputil import mp_pool_run
'''
common:
@ -121,18 +120,12 @@ class Explorer(object):
multiprocessing.get_start_method()))
self.log.debug(("Starting multiprocessing Pool for global "
"explorers run"))
with multiprocessing.Pool(self.jobs) as pool:
self.log.debug("Starting async for global explorer run")
results = [
pool.apply_async(self._run_global_explorer, (e, out_path,))
for e in self.list_global_explorer_names()
]
self.log.debug("Waiting async results for global explorer runs")
for r in results:
r.get() # self._run_global_explorer returns None
self.log.debug(("Multiprocessing run for global explorers "
"finished"))
args = [
(e, out_path, ) for e in self.list_global_explorer_names()
]
mp_pool_run(self._run_global_explorer, args, jobs=self.jobs)
self.log.debug(("Multiprocessing run for global explorers "
"finished"))
# logger is not pickable, so remove it when we pickle
def __getstate__(self):