I'm trying to do a multi-purpose paraloop class to be able to run multiprocessor jobs easily. Basically a user must define every iteration step as def iteration(index) within a with statement as in the example herein
here's my implementation
import multiprocessing as mp
import types
class paraloop(object):
def __init__(self, ncores, niterations):
self.niterations = niterations
self.ncores = min(ncores, self.niterations)
def __enter__(self, *args, **kwargs):
self.pool = mp.Pool(processes = self.ncores)
self.iterated = 0
# create results dict
self.result = {}
return self
def __exit__(self, *args, **kwargs):
print isinstance (iteration, types.MethodType)
def ITER():
self.iterated += 1
self.result[self.iterated] = iteration(self.iterated)
if self.iterated < self.niterations:
self.pool.apply_async( ITER ).get()
print isinstance (ITER, types.MethodType)
# run iterations in parallel
[self.pool.apply_async( ITER ).get() for idx in xrange(self.ncores)]
# usage example
import numpy as np
ITERATIONS = 10
ARRAY = np.ones(1000000)
with paraloop(ncores=4, niterations=ITERATIONS) as p:
def iteration(index):
print 'this is an iteration %i'%index
s = 0
for n in ARRAY:
s += n
return s
the print statement are to make sure that I have picklable function and not methods.
>> False
>> False
>> Traceback (most recent call last):
>> File "paraloop.py", line 48, in <module>
>> def iteration(index):
>> File "paraloop.py", line 29, in __exit__
>> [self.pool.apply_async( ITER ).get() for idx in xrange(self.ncores)]
>> File "c:\Python27\lib\multiprocessing\pool.py", line 558, in get
>> raise self._value
>> cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed
Any idea why I am having this error message ?
