I am trying to apply a function to each column of a dataframe using multiprocessing.
This was my approach based on Question 26784164
def function_that_takes_df(cls, df: pandas.DataFrame):
processes = multiprocessing.cpu_count() - 3
split_data = numpy.array_split(df, processes, axis=1)
with multiprocessing.Pool(processes) as p:
data = p.map(run_on_subset, split_data)
return pandas.concat(data, axis=1)
def run_on_subset(cls, data_subset) -> pandas.DataFrame:
x = data_subset.apply(function_that_takes_column)
return x
It works when I run the code, but if I try to debug it in Pycharm, I get:
Exception ignored in tp_clear of: <class 'memoryview'>
Traceback (most recent call last):
File "\venv\lib\site-packages\win32com\client\dynamic.py", line 173, in MakeOleRepr
olerepr = build.LazyDispatchItem(attr, None)
BufferError: memoryview has 1 exported buffer
