Python unicode encoding issue

Question

Using python 2.7.5. All databases and tables are

My code looks like that:

import MySQLdb as mdb
import urllib2
import sys
import logging
logging.basicConfig(level=logging.INFO)

from bs4 import BeautifulSoup as BS
con = mdb.connect('loclhost', 'root', '', 'mydb');
cur = con.cursor()
cur.execute('SET NAMES utf8;')
cur.execute('SET CHARACTER SET utf8;')
cur.execute('SET character_set_connection=utf8;')
with con:
...
        sql_insert = """INSERT INTO Teams (name, category, countryId) VALUES (%s, 1, %s)"""
        cursor = con.cursor()
        try:
            affected_count = cursor.execute(sql_insert, (name, id))  <<< this line
            con.commit()
        except mdb.IntegrityError:
            logging.warn("failed to insert values %s, %s", name, id)
        finally:
           cursor.close()
...

con.close()

Getting error message:

"UnicodeEncodeError: 'latin-1' codec can't encode character u'\u015f' in position 2: ordinal not in range(256)"

line marked above. What am i doing wrong?

Answer 1

Try:

con = mdb.connect('loclhost', 'root', '', 'mydb', 
                  use_unicode=True, charset='utf8')

---

Here is a demonstration showing that it works:

If you do not use use_unicode=True with the following setup, you get a UnicodeEncodeError:

import MySQLdb
import config

def setup_charset(cursor, typ='latin1'):
    sql = 'DROP TABLE IF EXISTS foo'
    cursor.execute(sql)
    sql = '''\
        CREATE TABLE `foo` (
          `fooid` int(11) NOT NULL AUTO_INCREMENT,
          `bar` varchar(30),
          `baz` varchar(30),
          PRIMARY KEY (`fooid`)) DEFAULT CHARSET={t}
        '''.format(t=typ)
    cursor.execute(sql)
    sql = 'INSERT INTO foo (bar,baz) VALUES (%s,%s)'

connection = MySQLdb.connect(
    host=config.HOST, user=config.USER,
    passwd=config.PASS, db='test')

cursor = connection.cursor()
setup_charset(cursor, typ='utf8')
sql = u'INSERT INTO foo (bar,baz) VALUES (%s,%s)'
try:
    cursor.execute(sql, [u'José Beiträge', u'∞'])
except UnicodeEncodeError as err:
    # You get this error if you don't use
    # (use_unicode=True, charset='utf8') see below.
    print(err)

raises the exception:

'latin-1' codec can't encode character u'\u221e' in position 0: ordinal not in range(256)

While, if you do use use_unicode=True, you can insert unicode with no error:

connection = MySQLdb.connect(
    host=config.HOST, user=config.USER,
    passwd=config.PASS, db='test',
    use_unicode=True,
    charset='utf8')
cursor = connection.cursor()
cursor.execute(sql, ['José Beiträge', '∞'])
cursor.execute('SELECT * from foo')
for row in cursor:
    print(u'{} {}'.format(*row[1:]))

prints

José Beiträge ∞