38

Importing pandas didn't throw the error, but rather trying to read a picked pandas dataframe as such:

import numpy as np
import pandas as pd
import matplotlib
import seaborn as sns
sns.set(style="white")

control_data = pd.read_pickle('null_report.pickle')
test_data = pd.read_pickle('test_report.pickle')

The traceback is 165 lines with three concurrent exceptions (whatever that means). Is read_pickle not compatible with pandas version 17.1 I'm running? How do I unpickle my dataframe for use?

Below is a copy of the traceback:

ImportError                               Traceback (most recent call last)
C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in try_read(path, encoding)
     45             with open(path, 'rb') as fh:
---> 46                 return pkl.load(fh)
     47         except (Exception) as e:

ImportError: No module named 'pandas.indexes'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in try_read(path, encoding)
     51                 with open(path, 'rb') as fh:
---> 52                     return pc.load(fh, encoding=encoding, compat=False)
     53 

C:\Users\test\Anaconda3\lib\site-packages\pandas\compat\pickle_compat.py in load(fh, encoding, compat, is_verbose)
    115 
--> 116         return up.load()
    117     except:

C:\Users\test\Anaconda3\lib\pickle.py in load(self)
   1038                 assert isinstance(key, bytes_types)
-> 1039                 dispatch[key[0]](self)
   1040         except _Stop as stopinst:

C:\Users\test\Anaconda3\lib\pickle.py in load_stack_global(self)
   1342             raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343         self.append(self.find_class(module, name))
   1344     dispatch[STACK_GLOBAL[0]] = load_stack_global

C:\Users\test\Anaconda3\lib\pickle.py in find_class(self, module, name)
   1383                 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384         __import__(module, level=0)
   1385         if self.proto >= 4:

ImportError: No module named 'pandas.indexes'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in read_pickle(path)
     59     try:
---> 60         return try_read(path)
     61     except:

C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in try_read(path, encoding)
     56                 with open(path, 'rb') as fh:
---> 57                     return pc.load(fh, encoding=encoding, compat=True)
     58 

C:\Users\test\Anaconda3\lib\site-packages\pandas\compat\pickle_compat.py in load(fh, encoding, compat, is_verbose)
    115 
--> 116         return up.load()
    117     except:

C:\Users\test\Anaconda3\lib\pickle.py in load(self)
   1038                 assert isinstance(key, bytes_types)
-> 1039                 dispatch[key[0]](self)
   1040         except _Stop as stopinst:

C:\Users\test\Anaconda3\lib\pickle.py in load_stack_global(self)
   1342             raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343         self.append(self.find_class(module, name))
   1344     dispatch[STACK_GLOBAL[0]] = load_stack_global

C:\Users\test\Anaconda3\lib\pickle.py in find_class(self, module, name)
   1383                 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384         __import__(module, level=0)
   1385         if self.proto >= 4:

ImportError: No module named 'pandas.indexes'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in try_read(path, encoding)
     45             with open(path, 'rb') as fh:
---> 46                 return pkl.load(fh)
     47         except (Exception) as e:

ImportError: No module named 'pandas.indexes'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in try_read(path, encoding)
     51                 with open(path, 'rb') as fh:
---> 52                     return pc.load(fh, encoding=encoding, compat=False)
     53 

C:\Users\test\Anaconda3\lib\site-packages\pandas\compat\pickle_compat.py in load(fh, encoding, compat, is_verbose)
    115 
--> 116         return up.load()
    117     except:

C:\Users\test\Anaconda3\lib\pickle.py in load(self)
   1038                 assert isinstance(key, bytes_types)
-> 1039                 dispatch[key[0]](self)
   1040         except _Stop as stopinst:

C:\Users\test\Anaconda3\lib\pickle.py in load_stack_global(self)
   1342             raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343         self.append(self.find_class(module, name))
   1344     dispatch[STACK_GLOBAL[0]] = load_stack_global

C:\Users\test\Anaconda3\lib\pickle.py in find_class(self, module, name)
   1383                 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384         __import__(module, level=0)
   1385         if self.proto >= 4:

ImportError: No module named 'pandas.indexes'

During handling of the above exception, another exception occurred:

ImportError                               Traceback (most recent call last)
<ipython-input-17-3b05fe7d20a4> in <module>()
      3 # test_data = np.genfromtxt(fh, usecols=2)
      4 
----> 5 control_data = pd.read_pickle('null_report.pickle')
      6 test_data = pd.read_pickle('test_report.pickle')
      7 

C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in read_pickle(path)
     61     except:
     62         if PY3:
---> 63             return try_read(path, encoding='latin1')
     64         raise

C:\Users\test\Anaconda3\lib\site-packages\pandas\io\pickle.py in try_read(path, encoding)
     55             except:
     56                 with open(path, 'rb') as fh:
---> 57                     return pc.load(fh, encoding=encoding, compat=True)
     58 
     59     try:

C:\Users\test\Anaconda3\lib\site-packages\pandas\compat\pickle_compat.py in load(fh, encoding, compat, is_verbose)
    114         up.is_verbose = is_verbose
    115 
--> 116         return up.load()
    117     except:
    118         raise

C:\Users\test\Anaconda3\lib\pickle.py in load(self)
   1037                     raise EOFError
   1038                 assert isinstance(key, bytes_types)
-> 1039                 dispatch[key[0]](self)
   1040         except _Stop as stopinst:
   1041             return stopinst.value

C:\Users\test\Anaconda3\lib\pickle.py in load_stack_global(self)
   1341         if type(name) is not str or type(module) is not str:
   1342             raise UnpicklingError("STACK_GLOBAL requires str")
-> 1343         self.append(self.find_class(module, name))
   1344     dispatch[STACK_GLOBAL[0]] = load_stack_global
   1345 

C:\Users\test\Anaconda3\lib\pickle.py in find_class(self, module, name)
   1382             elif module in _compat_pickle.IMPORT_MAPPING:
   1383                 module = _compat_pickle.IMPORT_MAPPING[module]
-> 1384         __import__(module, level=0)
   1385         if self.proto >= 4:
   1386             return _getattribute(sys.modules[module], name)[0]

ImportError: No module named 'pandas.indexes'

I also tried loading the pickle file from pickle directly:

via_pickle = pickle.load( open( 'null_report.pickle', "rb" ) )

and got the same error:

---------------------------------------------------------------------------
ImportError                               Traceback (most recent call last)
<ipython-input-23-ba2e3adae1c4> in <module>()
      1 
----> 2 via_pickle = pickle.load( open( 'null_report.pickle', "rb" ) )
      3 
      4 # control_data = pd.read_pickle('null_report.pickle')
      5 # test_data = pd.read_pickle('test_report.pickle')

ImportError: No module named 'pandas.indexes'
Thomas Matthew
  • 2,826
  • 4
  • 34
  • 58
  • 2
    I think you need to load an older version of pandas to unpickle this. Then save as csv or something else. Pickle depends on class structure and module pointing to reconstruct the object. If this structure changes, pickle will break. – piRSquared May 22 '16 at 07:20
  • 10
    you are trying to read a newer pickle with an older version of pandas. pandas.indexes doesn't exist until 0.18.0; pandas offers back compat not forward compat (meaning a newer versions can read older, but older cannot read newer) – Jeff May 22 '16 at 10:27
  • 5
    I am using 0.20.2, and have this problem with a pickle that was generated with an older version. – MPa Jun 29 '17 at 01:57
  • I'm using 0.20.3 and getting this error with an older pickle file too. – rer Aug 05 '17 at 18:18
  • 1
    @r3robertson try using `pd.read_pickle`, as indicated [here](https://github.com/pandas-dev/pandas/issues/16474#issuecomment-303676808) – miraculixx Sep 28 '17 at 16:15

7 Answers7

62

I had this error when I created a pkl file with python 2.7 and was trying to read it with python 3.6 I did:

pd.read_pickle('foo.pkl')

and it worked

Martin Tournoij
  • 26,737
  • 24
  • 105
  • 146
ricmarchao
  • 920
  • 8
  • 7
17

I had this problem from trying to open a pickled dataframe made with pandas 0.18.1 using pandas 0.17.1. If you are using pip, upgrade pandas with:

pip install --upgrade pandas

If you are using a library like anaconda, use:

conda upgrade pandas

If you need to have both versions of pandas on your machine, consider using virtualenv

Oliver
  • 189
  • 1
  • 6
  • 1
    Worked for me. It's a small detail, but I had to use pip3 install --upgrade pandas to ensure it worked on Python 3. – Berun Nov 27 '17 at 15:18
9

Saving and loading in different versions of pandas using pickle often does not work. Instead, use pandas.HDFStore.

When I needed to update pandas but also needed some data saved with pickle in previous versions, I went back and re-saved that data in HDF format instead, when nothing else would work. No problems anymore.

Works for any sort of pandas data structure it seems, even multi-indexed dataframes! In short, if pickling fails after a version upgrade, try HDFStore; it's more reliable (and more efficient!).

JoseOrtiz3
  • 1,785
  • 17
  • 28
  • Didn't even know about this method, which could perhaps also then be opened in `R`. – ryanjdillon Aug 25 '17 at 08:12
  • Thankyou for calling this to our attention. Just one nit, however. HDFStore was designed for pandas objects. If you want to store, say, a Python dictionary, you have to do some kudgey stuff, like store a stub DataFrame then add the dictionary object to it as an "attribute", as recommended in the pandas cookbook (http://pandas.pydata.org/pandas-docs/stable/cookbook.html#hdfstore). I personally never got this kludge to work. – John Strong Apr 16 '18 at 20:32
2

Here is the solution without updating pandas or whatever your using.

If you're using python2

import cPickle
with open('filename.pkl', 'rb') as fo:
        dict = cPickle.load(fo, encoding='latin1’)

If you're using python3

import pickle
with open('filename.pkl', 'rb') as fo:
        dict = pickle.load(fo, encoding='latin1’)
Gulzar
  • 23,452
  • 27
  • 113
  • 201
Dave.Cheng
  • 57
  • 1
  • 6
  • Your python3 solution doesn't work for me, it gives a `UnicodeDecodeError` on the last line,` 'utf-8' codec can't decode byte 0x80 in position 0: invalid start byte.` – user1416227 May 12 '17 at 19:54
  • 1
    still gives me ModuleNotFoundError: No module named 'pandas.indexes' on python3.6 – PdevG Sep 20 '17 at 10:05
1

In the pandas 0.23.4, there is a better way to fix the problem. Use the pandas.read_pickle to read the fileobject, like:

pd.read_pickle(open('test_report.pickle', 'rb'))

RayZen
  • 141
  • 1
  • 4
1

A flexible way to deal with internal API changes that break unpickling is to implement a custom Unpickler instance.

For example, the pandas.indexes module has been moved to pandas.core.indexes. We can write an Unpickler, that adapts the module path accordingly. To do that, we can overwrite the method find_class: 

import sys
class Unpickler(pickle.Unpickler):
    def find_class(self, module, name):
        '''This method gets called for every module pickle tries to load.'''
        # python 2 --> 3 compatibility: __builtin__ has been renamed to builtins
        if module == '__builtin__':
            module = 'builtins'
        # pandas compatibility: in newer versions, pandas.indexes has been moved to pandas.core.indexes
        if 'pandas.indexes' in module:
            module = module.replace('pandas.indexes', 'pandas.core.indexes')
        __import__(module)
        return getattr(sys.modules[module], name)

with open('/path/to/pickle.pkl', 'rb') as file:
    pdf = Unpickler(file).load()
Arco Bast
  • 3,595
  • 2
  • 26
  • 53
0

If you want to read pickled text instead of a file, do

import io

pd.read_pickle(io.BytesIO(pickled_text))

If you face the error - ValueError: Unrecognized compression type: infer,
explicitly mention the compression type. It could be one of None(no compression), gzip, bz2, xz or zip(depending upon file extension).
pd.read_pickle(io.BytesIO(pickled_text), compression=None)

Shiva
  • 2,627
  • 21
  • 33