python多处理不能腌制<type'function'> [重复]

Question

windows中的python2.7 |我将mysql连接添加到类中并使用多处理，引发错误。

self.ispop 和 self.match_var 返回字典

sprawn_self_calcu() 和 unwrap_self_f() 是 Map_class 函数的代理

Map_class 的功能需要自变量。

我的代码是这样的：

 from analysis_conf.pop_config import pop_config
 import datetime
 import multiprocessing
 from functools import  partial
 from sqlalchemy import create_engine
 from multiprocessing import Pool as threadpool

 def sprawn_self_calcu(arg, **kwarg):
      return Map.mapCin(*arg, **kwarg)

 def unwrap_self_f(arg, **kwarg):
      return Map.mappalg(*arg, **kwarg)
 partial_unwrap = partial(unwrap_self_f)
 partial_sprawn = partial(sprawn_self_calcu)

 class Map:
    def __init__(self):
        self.ispop = pop_config()
        self.match_var = self.ispop.pop_match_var()
    def CreateSqlalchemyEngine(self,config):
        sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'],
                                                         config['ipaddr'],config['port'],config['dbname']
                                                         )
        return create_engine(sigma,pool_recycle=10,pool_timeout=10800)

    def Mapping(self,conSet):
        self.baseCon = conSet
        self.mappalg()

        Time = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
    def IUCMapping(self,i,con):
        print i
        print self.conf
        l = con.execute('show tables;')

    def mappalg(self):
        mt_val = [1,2,3,4,5]
        pool = threadpool(4)
        result = pool.map(partial_sprawn,zip([self]*5,mt_val))
    # result = pool.map(partial_sprawn,zip([self]*mtlen,mt_val))
        pool.close()
        pool.join()
        return True

    def mapCin(self,i):
        pid_val = multiprocessing.current_process().pid%4
        con = self.baseCon[pid_val]
        print i
        self.IUCMapping(i,con)
        return True

 class Create_MultiCon:
    def __init__(self):
        self.adapter = pop_config()
        self.conf  = self.adapter.pop_baseDB()
        self.match_var = self.adapter.pop_match_var()
    def CreateSqlalchemyEngine(self,config):
        sigma = 'mysql+mysqldb://%s:%s@%s:%s/%s?charset=utf8'%(config['user'],config['passwd'],
                                                         config['ipaddr'],config['port'],config['dbname']
                                                         )
        return create_engine(sigma,pool_recycle=10,pool_timeout=10800)
def RdictXcon(self,x):
        t = {}
        engine = self.CreateSqlalchemyEngine(self.conf)
        for i in xrange(x):
            t[i] = engine.connect()
        return t

if __name__ == '__main__':
    l = Create_MultiCon()
    conSet = l.RdictXcon(4)
    ScMap =  Map()
    ScMap.Mapping(conSet)

错误：

    result = pool.map(partial_sprawn,zip([self]*5,mt_val))
  File "C:\Python27\lib\multiprocessing\pool.py", line 251, in map
    return self.map_async(func, iterable, chunksize).get()
  File "C:\Python27\lib\multiprocessing\pool.py", line 567, in get
    raise self._value
cPickle.PicklingError: Can't pickle <type 'function'>: attribute lookup __builtin__.function failed

如何解决错误

Answer 1

Python 的multiprocessing 模块无法处理无法腌制的函数/方法，这意味着您不能毫无麻烦地使用类或实例方法。我建议使用multiprocess，它使用dill而不是pickle进行序列化，并且可以处理类或实例方法。

据我所知，界面和multiprocessing中使用的完全一样，所以你可以直接用它代替。

另见https://stackoverflow.com/a/21345423/1170207