Since your asking for a canonical answer, here's an approach that avoids editing the provided object in-place, provides type hints, has a docstring, can be invoked with the remove parameter to remove different values, and errors with a TypeError as appropriate:
from collections import abc
from typing import Dict, List, Union
def without_nulls(d: Union[List, Dict], remove: tuple = ('NULL', 'null')) -> Union[List, Dict]:
""" Given a list or dict, returns an object of the same structure without null dict values. """
collection_types = (list, tuple) # avoid including 'str' here
mapping_types = (abc.Mapping,)
all_supported_types = (*mapping_types, *collection_types)
if isinstance(d, collection_types):
return [without_nulls(x, remove) if isinstance(x, all_supported_types) else x for x in d]
elif isinstance(d, mapping_types):
clean_val = lambda x: without_nulls(x, remove) if isinstance(x, all_supported_types) else x
return {k: clean_val(v) for k, v in d.items() if v not in remove}
raise TypeError(f"Unsupported type '{type(d)}': {d!r}")
Usage:
>>> cleaned_d = without_nulls(d)
>>> cleaned_d
[{'key1': 'value1', 'key2': {'key3': 'value3', 'z': {'z2': 'zzz'}}}, {'AAA': 'BBB'}]
>>> d # d is unchanged
[{'key1': 'value1', 'key2': {'key3': 'value3', 'key4': 'NULL', 'z': {'z1': 'NULL', 'z2': 'zzz'}}, 'key5': 'NULL'}, {'KEY': 'NULL', 'AAA': 'BBB'}]
If I wanted to have it remove None instead of 'NULL', I could call it like this:
without_nulls(d, remove=(None,))
Here's an even more generalized version - the sort of thing I'd expect to find in a general-purpose Python toolkit:
from collections import abc
from typing import Any, Callable, Dict, List, Optional, Union
def filter_dicts(d: Union[List, Dict],
value_filter: Optional[Callable[[Any], bool]] = None) -> Union[List, Dict]:
"""
Given a list or dict, returns an object of the same structure without filtered values.
By default, key-value pairs where the value is 'None' are removed. The `value_filter` param
must be a function which takes values from the provided dict/list structure, and returns a
truthy value if the key-value pair is to be removed, and a falsey value otherwise.
"""
collection_types = (list, tuple) # avoid including 'str' here
mapping_types = (abc.Mapping,)
all_supported_types = (*mapping_types, *collection_types)
if value_filter is None:
value_filter = lambda x: x is None
if isinstance(d, collection_types):
return [filter_dicts(x, value_filter) if isinstance(x, all_supported_types) else x for x in d]
elif isinstance(d, mapping_types):
clean_val = lambda x: filter_dicts(x, value_filter) if isinstance(x, all_supported_types) else x
return {k: clean_val(v) for k, v in d.items() if not value_filter(v)}
raise TypeError(f"Unsupported type '{type(d)}': {d!r}")
We can use it like this:
>>> filter_dicts(d)
[{'key1': 'value1', 'key2': {'key3': 'value3', 'key4': 'NULL', 'z': {'z1': 'NULL', 'z2': 'zzz'}}, 'key5': 'NULL'}, {'KEY': 'NULL', 'AAA': 'BBB'}]
>>> filter_dicts(d) == d
True
>>> filter_dicts(d, lambda x: x in (None, 'NULL', 'null'))
[{'key1': 'value1', 'key2': {'key3': 'value3', 'z': {'z2': 'zzz'}}}, {'AAA': 'BBB'}]
The benefit of this version is that it allows the caller to completely customize what criteria is employed when removing values. The function handles traversing the structure, and building up a new one to be returned, while the caller gets to decide what gets removed.
