I am trying to implement a shared counter in multiprocessing. I am using a global variable to get around pickling problems. For reasons I do not understand, the increment does not seem to apply on my global counter list (the value is always 0). I guess it the code is using a local instance of the variable that gets discarded.
I thought that global lists can be modified, because they are mutable. I also tried explicitly defining global list_global to specify that I want to use the global definition of the variable.
Can someone please point out my error?
from multiprocessing import Pool, Value, Lock
list_global = [] # global variable to hold Counter values
#http://eli.thegreenplace.net/2012/01/04/shared-counter-with-pythons-multiprocessing/
class Counter(object):
def __init__(self, initval=0):
self.val = Value('i', initval)
self.lock = Lock()
## self.val = initval
def increment(self):
with self.lock:
self.val.value += 1
def value(self):
with self.lock:
return self.val.value
def process_item(x):
global list_global
list_global[0].increment() # increments
return list_global[0].value() # correctly returns incremented value
def main():
global list_global
print 'before', list_global[0].value()
pool = Pool()
print pool.map(process_item, range(10))
pool.close()
pool.join()
#increments in process_item are not persistent
#(do not appear to be modifying the global variable)
print 'after', list_global[0].value() #=> 0
# list_global holds 3 Counter objects
for i in range(3):
list_global.append(Counter(0))
if __name__ == '__main__':
global list_global
main()
#print list_global # list_global holds "Counter" objects
for i in list_global:
print i.value(), #=>[0,0,0] # expected [10,0,0]
