I am dynamically creating classes using mixins, following the pattern in this SO answer, but it's giving me pickling errors:
https://stackoverflow.com/a/28205308/2056246
(In case anyone is curious why on earth I'd want to do this, it's really useful for machine learning when you might want to test different combinatorial pipelines of operations).
Here's a minimally reproducible example:
class AutoMixinMeta(ABCMeta):
"""
Helps us conditionally include Mixins, which is useful if we want to switch between different
combinations of models (ex. SBERT with Doc Embedding, RoBERTa with positional embeddings).
class Sub(metaclass = AutoMixinMeta):
def __init__(self, name):
self.name = name
"""
def __call__(cls, *args, **kwargs):
try:
mixin = kwargs.pop('mixin')
if isinstance(mixin, list):
mixin_names = list(map(lambda x: x.__name__, mixin))
mixin_name = '.'.join(mixin_names)
cls_list = tuple(mixin + [cls])
else:
mixin_name = mixin.__name__
cls_list = tuple([mixin, cls])
name = "{}With{}".format(cls.__name__, mixin_name)
cls = type(name, cls_list, dict(cls.__dict__))
except KeyError:
pass
return type.__call__(cls, *args, **kwargs)
class Mixer(metaclass = AutoMixinMeta):
""" Class to mix different elements in.
a = Mixer(config=config, mixin=[A, B, C])
"""
pass
class A():
pass
class B():
pass
config={
'test_a': True,
'test_b': True
}
def get_mixins(config):
mixins = []
if config['test_a']:
mixins.append(A)
if config['test_b']:
mixins.append(B)
return mixins
to_mix = get_mixins(config)
c = Mixer(mixin=to_mix)
import pickle
pickle.dump(c, open('tmp/test.pkl', 'wb'))
---------------------------------------------------------------------------
PicklingError Traceback (most recent call last)
<ipython-input-671-a7661c543ef8> in <module>
23
24 import pickle
---> 25 pickle.dump(c, open('tmp/test.pkl', 'wb'))
PicklingError: Can't pickle <class '__main__.MixerWithA.B'>: attribute lookup MixerWithA.B on __main__ failed
I've read several answers focusing on using __reduce__:
https://stackoverflow.com/a/11526524/2056246 https://stackoverflow.com/a/11493777/2056246 Pickling dynamically created classes
But none of them deal with this dynamic creation on the metaclass level.
Has anyone run into this problem before/have any idea how to solve it? I don't know enough about metaclasses unfortunately to understand how to approach this problem.
