I have lambda function that is screwing up the pickling of an object. What makes it hard to debug is that it doesn't tell me the name of field causing this issue. I created a recursive function that tries to find such fields but it fails on the code I need it to work on (but succeeds in toy self contained cases).
It works in this toy example:
# %%
"""
trying to detect which field is the anonymous function giving me isse since:
AttributeError: Can't pickle local object 'FullOmniglot.__init__.<locals>.<lambda>'
doesn't tell me which one for some reason.
"""
import re
from typing import Any, Callable, Union, Optional
def _is_anonymous_function(f) -> bool:
"""
Returns true if it's an anonynouys function.
ref: https://stackoverflow.com/questions/3655842/how-can-i-test-whether-a-variable-holds-a-lambda
"""
return callable(f) and f.__name__ == "<lambda>"
def _get_anonymous_function_attributes(anything, halt: bool = False, verbose: bool = False) -> dict:
"""
Returns the dictionary of name of fields to anonymous functions in the past anything thing.
:param anything:
:param halt:
:param verbose:
:return:
"""
anons: dict = {}
for field_name in dir(anything):
field = getattr(anything, field_name)
if _is_anonymous_function(field):
if verbose:
print(f'{field_name=}')
print(f'{field=}')
if halt:
from pdb import set_trace as st
st()
anons[str(field_name)] = field
return anons
def _get_anonymous_function_attributes_recursive(anything: Any, path: str = '') -> dict[str, Callable]:
""""""
anons: dict = {}
def __get_anonymous_function_attributes_recursive(anything: Any,
path: Optional[str] = '',
) -> None:
if _is_anonymous_function(anything):
# assert field is anything, f'Err not save thing/obj: \n{field=}\n{anything=}'
# key: str = str(dict(obj=anything, field_name=field_name))
key: str = str(path)
anons[key] = anything
else:
for field_name in dir(anything):
# if field_name != '__abstractmethods__':
if not bool(re.search(r'__(.+)__', field_name)):
field = getattr(anything, field_name)
# only recurse if new field is not itself
if field is not anything: # avoids infinite recursions
path_for_this_field = f'{path}.{field_name}'
__get_anonymous_function_attributes_recursive(field, path_for_this_field)
return
__get_anonymous_function_attributes_recursive(anything, path)
return anons
class MyObj:
def __init__(self):
self.data = 'hi'
self.anon = lambda x: x
local_variable_me = 'my a local variable!'
def non_anon(self, x):
return x
class MyObj2:
def __init__(self):
self.data = 'hi'
self.anon = lambda x: x
local_variable_me = 'my a local variable!'
self.obj = MyObj()
def non_anon(self, x):
return x
"""
Trying to fix: AttributeError: Can't pickle local object 'FullOmniglot.__init__.<locals>.<lambda>'
Trying to approximate with my obj and get: obj.__init__.<locals> to to get the obj.__ini__.<locals>.<lambda>
"""
top_obj = MyObj2()
# print(f'anons recursive: {_get_anonymous_function_attributes_recursive(obj)=}')
print('getting all anonymous functions recursively: ')
anons: dict = _get_anonymous_function_attributes_recursive(top_obj, 'top_obj')
print(f'{len(anons.keys())=}')
for k, v in anons.items():
print()
print(f'{k=}')
print(f'{v=}')
# print(k, v)
print()
but fails in the wild pytorch code:
# %%
"""
pip install torch
pip install learn2learn
"""
print()
import learn2learn
from torch.utils.data import DataLoader
omni = learn2learn.vision.benchmarks.get_tasksets('omniglot', root='~/data/l2l_data')
loader = DataLoader(omni, num_workers=1)
next(iter(loader))
print()
with error:
Traceback (most recent call last):
File "/Users/brandomiranda/opt/anaconda3/envs/meta_learning/lib/python3.9/multiprocessing/popen_spawn_posix.py", line 47, in _launch
reduction.dump(process_obj, fp)
File "/Users/brandomiranda/opt/anaconda3/envs/meta_learning/lib/python3.9/multiprocessing/reduction.py", line 60, in dump
ForkingPickler(file, protocol).dump(obj)
AttributeError: Can't pickle local object 'FullOmniglot.__init__.<locals>.<lambda>'
Why does it fail there?
Full self contained reproducible code in one place:
# %%
"""
trying to detect which field is the anonymous function giving me isse since:
AttributeError: Can't pickle local object 'FullOmniglot.__init__.<locals>.<lambda>'
doesn't tell me which one for some reason.
"""
import re
from typing import Any, Callable, Union, Optional
def _is_anonymous_function(f) -> bool:
"""
Returns true if it's an anonynouys function.
ref: https://stackoverflow.com/questions/3655842/how-can-i-test-whether-a-variable-holds-a-lambda
"""
return callable(f) and f.__name__ == "<lambda>"
def _get_anonymous_function_attributes(anything, halt: bool = False, verbose: bool = False) -> dict:
"""
Returns the dictionary of name of fields to anonymous functions in the past anything thing.
:param anything:
:param halt:
:param verbose:
:return:
"""
anons: dict = {}
for field_name in dir(anything):
field = getattr(anything, field_name)
if _is_anonymous_function(field):
if verbose:
print(f'{field_name=}')
print(f'{field=}')
if halt:
from pdb import set_trace as st
st()
anons[str(field_name)] = field
return anons
def _get_anonymous_function_attributes_recursive(anything: Any, path: str = '') -> dict[str, Callable]:
""""""
anons: dict = {}
def __get_anonymous_function_attributes_recursive(anything: Any,
path: Optional[str] = '',
) -> None:
if _is_anonymous_function(anything):
# assert field is anything, f'Err not save thing/obj: \n{field=}\n{anything=}'
# key: str = str(dict(obj=anything, field_name=field_name))
key: str = str(path)
anons[key] = anything
else:
for field_name in dir(anything):
# if field_name != '__abstractmethods__':
if not bool(re.search(r'__(.+)__', field_name)):
field = getattr(anything, field_name)
# only recurse if new field is not itself
if field is not anything: # avoids infinite recursions
path_for_this_field = f'{path}.{field_name}'
__get_anonymous_function_attributes_recursive(field, path_for_this_field)
return
__get_anonymous_function_attributes_recursive(anything, path)
return anons
class MyObj:
def __init__(self):
self.data = 'hi'
self.anon = lambda x: x
local_variable_me = 'my a local variable!'
def non_anon(self, x):
return x
class MyObj2:
def __init__(self):
self.data = 'hi'
self.anon = lambda x: x
local_variable_me = 'my a local variable!'
self.obj = MyObj()
def non_anon(self, x):
return x
"""
Trying to fix: AttributeError: Can't pickle local object 'FullOmniglot.__init__.<locals>.<lambda>'
Trying to approximate with my obj and get: obj.__init__.<locals> to to get the obj.__ini__.<locals>.<lambda>
"""
top_obj = MyObj2()
# print(f'anons recursive: {_get_anonymous_function_attributes_recursive(obj)=}')
print('getting all anonymous functions recursively: ')
anons: dict = _get_anonymous_function_attributes_recursive(top_obj, 'top_obj')
print(f'{len(anons.keys())=}')
for k, v in anons.items():
print()
print(f'{k=}')
print(f'{v=}')
# print(k, v)
print()
# from uutils import get_anonymous_function_attributes_recursive
# get_anonymous_function_attributes_recursive(top_obj, 'top_obj', print_output=True)
# print()
# %%
"""
pip install torch
pip install learn2learn
"""
print()
import learn2learn
from torch.utils.data import DataLoader
omni = learn2learn.vision.benchmarks.get_tasksets('omniglot', root='~/data/l2l_data')
loader = DataLoader(omni, num_workers=1)
next(iter(loader))
print()
related:
- Error involving lambda giving code <function main.<locals>.<lambda> at 0x00000234D43C68B8>
- Error involving lambda giving code <function main.<locals>.<lambda> at 0x00000234D43C68B8>
- AttributeError: Can't pickle local object '<locals>.<lambda>'
- How can I get the values of the locals of a function after it has been executed?
- l2l gitissue: https://github.com/learnables/learn2learn/issues/369
