I have a written a class to process data from a specific instrument. Each instance of the class contains a different example of this data. An equivalent toy example of this is given in the code below, which is a modified/mutilated version of this example:
import pickle as pkl
from multiprocessing import Process, Queue, current_process, freeze_support
class TestMaths:
def __init__(self, a, b):
self.a = a
self.b = b
self.c = None
def mul(self):
self.c = self.a * self.b
return 0
def plus(self):
self.c = self.a + self.b
return 0
def worker(input, output):
for maths_class in iter(input.get, 'STOP'):
result = calculate(maths_class)
output.put(maths_class)
def calculate(maths_class):
result = maths_class.mul()
return "calculation returns {0:d}".format(result)
def test():
NUMBER_OF_PROCESSES = 4
maths_classes = [TestMaths(ii, 7) for ii in range(20)]
TASKS1 = [(maths_class) for maths_class in maths_classes]
TASKS2 = [(maths_class) for maths_class in maths_classes]
# Create queues
task_queue = Queue()
done_queue = Queue()
# Submit tasks
for task in TASKS1:
task_queue.put(task)
# Start worker processes
for i in range(NUMBER_OF_PROCESSES):
Process(target=worker, args=(task_queue, done_queue)).start()
# Get and print results
print('Unordered results:')
for i in range(len(TASKS1)):
print('\t', done_queue.get().c)
# Tell child processes to stop
for i in range(NUMBER_OF_PROCESSES):
task_queue.put('STOP')
if __name__ == '__main__':
freeze_support()
# confirm pickleable
test_pickle = TestMaths(2, 2)
test_pickle.mul()
print("{0:f} x {1:f} = {2:f}".format(test_pickle.a, test_pickle.b, test_pickle.c))
pkl.dump(test_pickle, open("test.pkl", "wb"))
test_pickle_returned = pkl.load(open("test.pkl", "rb"))
print("{0:f} x {1:f} = {2:f}".format(test_pickle_returned.a, test_pickle_returned.b, test_pickle_returned.c))
# parallelisation
test()
The first block of code under the main if statement is supposed to confirm that the class is pickleable. After this the call to the function test runs the actually toy problem. When I run this with Python 3.6.6 on Windows 7 I get the following error:
Traceback (most recent call last):
File "C:\Users\rpoolman\Documents\QAS Results\Objective2\analysis\test2.py", line 82, in <module>
test()
File "C:\Users\rpoolman\Documents\QAS Results\Objective2\analysis\test2.py", line 62, in test
print('\t', done_queue.get().c)
File "C:\Users\rpoolman\WinPython-64bit-3.6.6.1Qt5\python-3.6.6.amd64\lib\multiprocessing\queues.py", line 113, in get
return _ForkingPickler.loads(res)
AttributeError: Can't get attribute 'TestMaths' on <module '__main__' from 'C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Professional\\Common7\\IDE\\Extensions\\Microsoft\\Python\\Core\\ptvsd_launcher.py'>
I'm aware the class needs to be Pickeable and think that TestMaths is. However, it seems that this error relates to the multiprocessing module's version of Pickle. Is there a way to refactor my class so that I can use multiprocessing?
