I would like to create an OrderedDict
instance that can be shared across processes.
What I've tried
Intro
I've implemented a normal dict
version, a multi-process dict
version, as shown below.
import multiprocessing as mp
class BasicPen(object):
"""single process version"""
def __init__(self, words):
self.dogs = {"Molson": [], "Taco": [], "Lucky": []}
self.punc = "!"
for dog_name in self.dogs.keys():
self.bark(dog_name, words)
def bark(self, dog, words):
for word in words:
self.dogs[dog].append(word + self.punc)
class MultiPen(object):
"""multiprocess dict"""
def __init__(self, words):
self.man = mp.Manager()
self.dogs = self.man.dict()
self.dogs["Molson"] = self.man.list()
self.dogs["Taco"] = self.man.list()
self.dogs["Lucky"] = self.man.list()
self.punc = "!"
procs = [mp.Process(target=self.bark, args=(dog_name, words)) for dog_name in self.dogs.keys()]
for pr in procs:
pr.start()
for pr in procs:
pr.join()
def bark(self, dog, words):
for word in words:
self.dogs[dog] += [word + self.punc]
if __name__ == '__main__':
print("multi")
m_pen = MultiPen(["bark", "woof", "yip"])
print(m_pen.dogs)
print("basic")
b_pen = BasicPen(["bark", "woof", "yip"])
print(b_pen.dogs)
Using multiprocessing.Namespace
I tried to implement a version with an OrderedDict
version using multiprocessing.Namespace()
as shown below:
import multiprocessing as mp
from collections import OrderedDict
class NamePen(object):
"""ordered dict attempt"""
def __init__(self, words):
man = mp.Manager()
self.ns = man.Namespace()
self.ns.dogs = OrderedDict()
self.ns.dogs["Lucky"] = man.list()
self.ns.dogs["Molson"] = man.list()
self.ns.dogs["Taco"] = man.list()
self.punc = "!"
procs = [mp.Process(target=self.bark, args=(dog_name, words)) for dog_name in self.ns.dogs.keys()]
for pr in procs:
pr.start()
for pr in procs:
pr.join()
def bark(self, dog, words):
for word in words:
self.ns.dogs[dog] += [word + self.punc]
if __name__ == '__main__':
print("ordered")
n_pen = NamePen(["bark", "woof", "yip"])
print(n_pen.ns.dogs)
When I run this code, the NamePen
return an empty OrderedDict
. An alternative I considered was creating a multiprocessing.Namespace
and just assigning attributes as dictionary keys, but then I couldn't find a way to list the attributes, since running __dict__
on a multiprocessing.Namespace
doesn't work. Consequently, I'm starting to doubt in the multiprocessing.Namespace
approach is correct.
Using multiprocessing.managers.BaseManager
After reading through questions about giving multi-process access to object instances, I tried to use the recommended approach of the BaseManager
:
import multiprocessing as mp
from multiprocessing.managers import BaseManager
from collections import OrderedDict
class MultiPen(object):
def __init__(self, words):
man = BaseManager()
man.register('OrderedDict', OrderedDict)
man.start()
self.dogs = man.OrderedDict()
self.dogs.update(molson=[])
self.dogs.update(taco=[])
self.dogs.update(lucky=[])
self.punc = "!"
procs = [mp.Process(target=self.bark, args=(dog_name, words)) for dog_name in self.dogs.keys()]
for pr in procs:
pr.start()
for pr in procs:
pr.join()
def bark(self, dog, words):
for word in words:
self.dogs[dog] += [word + self.punc]
if __name__ == '__main__':
print("multi")
m_pen = MultiPen(["bark", "woof", "yip"])
print(m_pen.dogs)
However, I get the following error:
Traceback (most recent call last):
File "multiproc_class.py", line 43, in <module>
m_pen = MultiPen(["bark", "woof", "yip"])
File "multiproc_class.py", line 17, in __init__
procs = [mp.Process(target=self.bark, args=(dog_name, words)) for dog_name in self.dogs.keys()]
File "<string>", line 2, in keys
File "/usr/lib/python3.5/multiprocessing/managers.py", line 732, in _callmethod
raise convert_to_error(kind, result)
multiprocessing.managers.RemoteError:
---------------------------------------------------------------------------
Unserializable message: ('#RETURN', odict_keys(['molson', 'taco', 'lucky']))
---------------------------------------------------------------------------
I think this implies that I'm using the object manager incorrectly.
What is an acceptable answer
I don't care if I need to use a library or a custom class. However, the OrderedDict does have to be an attribute of the class, as shown in the above examples.