Skip to content
Snippets Groups Projects
Commit 047110c0 authored by Bryan O'Sullivan's avatar Bryan O'Sullivan
Browse files

worker: allow a function to be run in multiple worker processes

If we estimate that it will be worth the cost, we run the function in
multiple processes. Otherwise, we run it in-process.

Children report progress to the parent through a pipe.

Not yet implemented on Windows.
parent ac4dbcee
No related branches found
No related tags found
No related merge requests found
......@@ -6,7 +6,7 @@
# GNU General Public License version 2 or any later version.
from i18n import _
import os, util
import os, signal, sys, util
def countcpus():
'''try to count the number of CPUs on the system'''
......@@ -53,6 +53,62 @@
benefit = linear - (_startupcost * workers + linear / workers)
return benefit >= 0.15
def worker(ui, costperarg, func, staticargs, args):
'''run a function, possibly in parallel in multiple worker
processes.
returns a progress iterator
costperarg - cost of a single task
func - function to run
staticargs - arguments to pass to every invocation of the function
args - arguments to split into chunks, to pass to individual
workers
'''
if worthwhile(ui, costperarg, len(args)):
return _platformworker(ui, func, staticargs, args)
return func(*staticargs + (args,))
def _posixworker(ui, func, staticargs, args):
rfd, wfd = os.pipe()
workers = _numworkers(ui)
for pargs in partition(args, workers):
pid = os.fork()
if pid == 0:
try:
os.close(rfd)
for i, item in func(*(staticargs + (pargs,))):
os.write(wfd, '%d %s\n' % (i, item))
os._exit(0)
except KeyboardInterrupt:
os._exit(255)
os.close(wfd)
fp = os.fdopen(rfd, 'rb', 0)
oldhandler = signal.getsignal(signal.SIGINT)
signal.signal(signal.SIGINT, signal.SIG_IGN)
def cleanup():
# python 2.4 is too dumb for try/yield/finally
signal.signal(signal.SIGINT, oldhandler)
problems = 0
for i in xrange(workers):
problems |= os.wait()[1]
if problems:
sys.exit(1)
try:
for line in fp:
l = line.split(' ', 1)
yield int(l[0]), l[1][:-1]
except: # re-raises
cleanup()
raise
cleanup()
if os.name != 'nt':
_platformworker = _posixworker
def partition(lst, nslices):
'''partition a list into N slices of equal size'''
n = len(lst)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment