Converting Unicode strings to Ascii strings, in a dict

Question

I read the answer to this question and still am getting the error AttributeError: 'dict' object has no attribute 'encode'.

I've tried

dic = pickle.load(fileObject)
for v in dic:
    v.encode('ascii', 'ignore')

and

dic = pickle.load(fileObject)
for key, val in dic.iteritems():
    val.encode('ascii', 'ignore')

and still get the same error. When printing out the variables they all display with a u at the front. The dictionary was pickled under python 3 and is being unpickled in python 2.

I tried pp.pprint((dataFromPrevMod).encode('ascii', 'ignore')) and it didn't work.

If I pprint out the dictionary, it shows its contents but in Python 3 each line starting with a u for example u'website': u'exmample.org'

Dictionary pretty printed in Python 3

{
        'output': {
                'table': 'intersection',
                'file_location': '\\\\storage1\\tpn\\tpn_team\\dev\\asmithe\\',
                'schema': 'asmithe',
                'temporary_location': '\\\\storage1\\tpn\\tpn_team\\dev\\asmithe\
\'
        },
        'tpn_inventory_db_r': {
                'generic_pwd': '51f3tlNE26',
                'db_name': 'tpn',
                'user': 'asmithe',
                'schema': 'asmithe',
                'host': 'example.tpns.org'
        },
        'proj_year': '2005',
        'proj_rules_r': 'C:\\asmithe\\rules.txt',
        'incidents_db_r': {
                'schema': 'tpn_pp_dist',
                'generic_pwd': '51f3tlNE26',
                'db_name': 'tpn',
                'user': 'asmithe',
                'fire_table': 'incident',
                'host': 'example.tpns.org'
        },
        'plots_to_project_r': 'C:\\Users\\asmithe\\Plots.txt',
        'tpn_proj_db_r': {
                'generic_pwd': '51f3tlNE26',
                'db_name': 'tpn inventory',
                'user': 'asmithe',
                'schema': 'test',
                'host': 'example.tpns.org'
        }
}

Dictionary pretty printed in Python 2 (notice the addition of u)

{   u'incidents_db_r': {   u'db_name': u'tpn',
                                u'fire_table': u'incident',
                                u'generic_pwd': u'51f3tlNE26',
                                u'host': u'example.tpns.org',
                                u'schema': u'tpn_pp_dist',
                                u'user': u'asmithe'},
    u'tpn_inventory_db_r': {   u'db_name': u'tpn',
                                 u'generic_pwd': u'51f3tlNE26',
                                 u'host': u'example.tpns.org',
                                 u'schema': u'asmithe',
                                 u'user': u'asmithe'},
    u'tpn_proj_db_r': {   u'db_name': u'tpn inventory',
                            u'generic_pwd': u'51f3tlNE26',
                            u'host': u'example.tpns.org',
                            u'schema': u'test',
                            u'user': u'asmithe'},
    u'output': {   u'file_location': u'\\\\storage1\\tpn\\tpn_team\\dev\\asmithe\ \',
                   u'schema': u'asmithe',
                   u'table': u'intersection',
                   u'temporary_location': u'\\\\storage1\\tpn\\tpn_team\\dev\\asmithe idek\\'},
    u'plots_to_project_r': u'C:\\Users\\asmithe\\Plots.txt',
    u'proj_rules_r': u'C:\\asmithe\\rules.txt',
    u'proj_year': u'2005'}

Answer 1

The reason the pprint shows unicode strings as u'bla' is that in Python 2, string and unicode objects are both sequence types, but they are not the same.

So it is logical that pprint in Python 2 doesn't show them as such.

Answer 2

You have nested dicts, so you need something like this (done very quickly just to give you an idea):

def unicode_to_bytes(d):
    for k in d:
        if isinstance(d[k], dict):
            unicode_to_bytes(d[k])
        elif isinstance(d[k], unicode):
            d[k] = d[k].encode('ascii', 'ignore')

test = {
    'a': u'b',
    'b': {
        'c': u'c',
        'd': {'e': u'f'}
    }
}

unicode_to_bytes(test)

print test

This is not taking care of keys though.

Hope it helps.

Answer 3

For the sake of completeness: It's astoundingly complex to write a custom pickler that automatically encodes unicode objects to strings. The below is for Python 2 only:

import pickle
import sys
from StringIO import StringIO

class EncodingUnpickler(pickle.Unpickler):
    def __init__(self, f, encoding=None):
        pickle.Unpickler.__init__(self, f)
        # we don't want to modify the class variable from pickle.Unpickler
        self.dispatch = self.dispatch.copy()
        self.dispatch[pickle.UNICODE] = EncodingUnpickler.load_unicode
        self.dispatch[pickle.BINUNICODE] = EncodingUnpickler.load_binunicode
        self.encoding = encoding or sys.getdefaultencoding()

    def load_binunicode(self):
        pickle.Unpickler.load_binunicode(self)
        self.append(self.stack.pop().encode(self.encoding))

    def load_unicode(self):
        pickle.Unpickler.load_unicode(self)
        self.append(self.stack.pop().encode(self.encoding))


d = { u'1': u'a', u'2': u'b' }
s = pickle.dumps(d)
unp = EncodingUnpickler(StringIO(s))
du = unp.load()
print du