python multiprocessing Can't pickle

Question

python2.7 in windows | i add the mysql connect into the class and use multiprocessing ,raise the error .

self.ispop and self.match_var return the dict

sprawn_self_calcu() and unwrap_self_f() is a proxy for the function of Map_class

the function of Map_class need self var.

my code like this:

 from analysis_conf.pop_config import pop_config
 import datetime
 import multiprocessing
 from functools import  partial
 from sqlalchemy import create_engine
 from multiprocessing import Pool as threadpool

 def sprawn_self_calcu(arg, **kwarg):
      return Map.mapCin(*arg, **kwarg)

 def unwrap_self_f(arg, **kwarg):
      return Map.mappalg(*arg, **kwarg)
 partial_unwrap = partial(unwrap_self_f)
 partial_sprawn = partial(sprawn_self_calcu)

 class Map:
    def __init__(self):
        self.ispop = pop_config()
        self.match_var = self.ispop.pop_match_var()
    def CreateSqlalchemyEngine(self,config):
        sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'],
                                                         config['ipaddr'],config['port'],config['dbname']
                                                         )
        return create_engine(sigma,pool_recycle=10,pool_timeout=10800)

    def Mapping(self,conSet):
        self.baseCon = conSet
        self.mappalg()

        Time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    def IUCMapping(self,i,con):
        print i
        print self.conf
        l = con.execute('show tables;')

    def mappalg(self):
        mt_val = [1,2,3,4,5]
        pool = threadpool(4)
        result = pool.map(partial_sprawn,zip([self]*5,mt_val))
    # result = pool.map(partial_sprawn,zip([self]*mtlen,mt_val))
        pool.close()
        pool.join()
        return True

    def mapCin(self,i):
        pid_val = multiprocessing.current_process().pid%4
        con = self.baseCon[pid_val]
        print i
        self.IUCMapping(i,con)
        return True

 class Create_MultiCon:
    def __init__(self):
        self.adapter = pop_config()
        self.conf  = self.adapter.pop_baseDB()
        self.match_var = self.adapter.pop_match_var()
    def CreateSqlalchemyEngine(self,config):
        sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'],
                                                         config['ipaddr'],config['port'],config['dbname']
                                                         )
        return create_engine(sigma,pool_recycle=10,pool_timeout=10800)
def RdictXcon(self,x):
        t = {}
        engine = self.CreateSqlalchemyEngine(self.conf)
        for i in xrange(x):
            t[i] = engine.connect()
        return t

if __name__ == '__main__':
    l = Create_MultiCon()
    conSet = l.RdictXcon(4)
    ScMap =  Map()
    ScMap.Mapping(conSet)

the Error :

    result = pool.map(partial_sprawn,zip([self]*5,mt_val))
  File "C:\Python27\lib\multiprocessing\pool.py", line 251, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 567, in get
    raise self._value
cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed

how to solv the ERROR

Answer 1

Python's multiprocessing module can not deal with functions/methods which cannot be pickled, which means you cannot use class or instance methods without a lot of hassle. I would recommend to use multiprocess, which uses dill for serialization instead of pickle, and can deal with class or instance methods.

As far as I know, the interface is exactly the same as the one used in multiprocessing, so you can use it as a drop-in replacement.

See also https://stackoverflow.com/a/21345423/1170207