How to compare two list of nested dicts in python3 having string in value

Question

I'm trying to compare two list of dicts in Python 3 and it's not working. They're both essentially the same but they're out of order.

test3 = [{'item': {'locations': '{"l":[{"m":{"officeHours":{"m":{"0":{"s":"00:00-00:00"},"1":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"3":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"5":{"s":"00:00-00:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"vid":{"s":"123456781234561234"},"accessible":{"n":"1"},"formatted":{"s":"Address"},"ordinal":{"n":"2"},"primary":{"s":"N"}}},"geolocation":{"m":{"latitude":{"n":"42.054029999999997"},"longitude":{"n":"-87.888379999999998"}}}}},{"m":{"officeHours":{"m":{"0":{"s":"00:00-00:00"},"1":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"3":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"5":{"s":"00:00-00:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"vid":{"s":"243354971448280066"},"accessible":{"n":"1"},"formatted":{"s":"10105 74th St Ste 101 Atlanta WI 53142"},"ordinal":{"n":"1"},"primary":{"s":"Y"}}},"geolocation":{"m":{"latitude":{"n":"42.567709999999998"},"longitude":{"n":"-87.930769999999995"}}}}}]}', 'name': '{"m":{"last":{"s":"Cernak"},"first":{"s":"Cynthia"}}}', 'priority': '{"n":"0"}', 'flashInTargeting': '{"n":"1"}', 'flashVisible': '{"n":"1"}', 'lastModified': '{"n":"1234"}', 'pna': '{"s":"N"}', 'specialty': '{"m":{"code":{"s":"POD"},"desc":{"s":"DESC"}}}', 'employeeId': '{"s":"15030"}', 'profession': '{"m":{"code":{"s":"8"},"desc":{"s":"DESC"}}}', 'inboundTrx': '{"l":[{"m":{"employeeName":{"s":"Test Rep"},"commTrxL7d":{"n":"1"},"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX"},"cashTrxL7d":{"n":"0"},"productName":{"s":"PN"},"commTrxL1d":{"n":"0"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"}}},{"m":{"employeeName":{"s":"Test Rep"},"commTrxL7d":{"n":"1"},"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"1"},"productName":{"s":"PN A"},"commTrxL1d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"}}}]}', 'npi': '{"s":"1467493197"}'}}, {'item': {'locations': '{"l":[{"m":{"officeHours":{"m":{"0":{"s":"00:00-00:00"},"1":{"s":"00:00-00:00"},"2":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"4":{"s":"00:00-00:00"},"5":{"s":"00:00-00:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"vid":{"s":"0"},"accessible":{"n":"1"},"formatted":{"s":"UNKNOWN ADDRESS"},"ordinal":{"n":"99"},"primary":{"s":"N"}}},"geolocation":{"m":{"latitude":{"n":"0"},"longitude":{"n":"0"}}}}}]}', 'name': '{"m":{"last":{"s":"PDRP"},"first":{"s":"PDRP"}}}', 'priority': '{"n":"0"}', 'flashInTargeting': '{"n":"0"}', 'flashVisible': '{"n":"1"}', 'lastModified': '{"n":"1234"}', 'pna': '{"s":"N"}', 'specialty': '{"m":{"code":{"s":"UNKNOWN SPECIALTY CODE"},"desc":{"s":"UNKNOWN SPECIALTY DESC"}}}', 'employeeId': '{"s":"15030"}', 'profession': '{"m":{"code":{"s":"UNKNOWN PROFESSION CODE"},"desc":{"s":"UNKNOWN PROFESSION DESC"}}}', 'inboundTrx': '{"l":[{"m":{"employeeName":{"s":"Test Rep"},"commTrxL7d":{"n":"1"},"reportDate":{"s":"2017-10-12"},"city":{"s":"PDRP"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"0"},"productName":{"s":"PDRP"},"commTrxL1d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"}}}]}', 'npi': '{"s":"PDRP"}'}}]

true3 = [{'item': {'employeeId': '{"s":"15030"}', 'flashInTargeting': '{"n":"1"}', 'flashVisible': '{"n":"1"}', 'inboundTrx': '{"l":[{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"commTrxL1d":{"n":"0"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX"},"cashTrxL7d":{"n":"0"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PN"},"employeeName":{"s":"Test Rep"}}},{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"commTrxL1d":{"n":"1"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"1"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PN A"},"employeeName":{"s":"Test Rep"}}}]}', 'lastModified': '{"n":"1234"}', 'locations': '{"l":[{"m":{"geolocation":{"m":{"latitude":{"n":"42.054029999999997"},"longitude":{"n":"-87.888379999999998"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"5":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"2"},"formatted":{"s":"Address"},"vid":{"s":"123456781234561234"},"primary":{"s":"N"}}}}},{"m":{"geolocation":{"m":{"latitude":{"n":"42.567709999999998"},"longitude":{"n":"-87.930769999999995"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"5":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"1"},"formatted":{"s":"10105 74th St Ste 101 Atlanta WI 53142"},"vid":{"s":"243354971448280066"},"primary":{"s":"Y"}}}}}]}', 'name': '{"m":{"last":{"s":"Cernak"},"first":{"s":"Cynthia"}}}', 'npi': '{"s":"1467493197"}', 'pna': '{"s":"N"}', 'priority': '{"n":"0"}', 'profession': '{"m":{"code":{"s":"8"},"desc":{"s":"DESC"}}}', 'specialty': '{"m":{"code":{"s":"POD"},"desc":{"s":"DESC"}}}'}}, {'item': {'employeeId': '{"s":"15030"}', 'flashInTargeting': '{"n":"0"}', 'flashVisible': '{"n":"1"}', 'inboundTrx': '{"l":[{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"PDRP"},"commTrxL1d":{"n":"1"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"0"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PDRP"},"employeeName":{"s":"Test Rep"}}}]}', 'lastModified': '{"n":"1234"}', 'locations': '{"l":[{"m":{"geolocation":{"m":{"latitude":{"n":"0"},"longitude":{"n":"0"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"00:00-00:00"},"5":{"s":"00:00-00:00"},"4":{"s":"00:00-00:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"99"},"formatted":{"s":"UNKNOWN ADDRESS"},"vid":{"s":"0"},"primary":{"s":"N"}}}}}]}', 'name': '{"m":{"last":{"s":"PDRP"},"first":{"s":"PDRP"}}}', 'npi': '{"s":"PDRP"}', 'pna': '{"s":"N"}', 'priority': '{"n":"0"}', 'profession': '{"m":{"code":{"s":"UNKNOWN PROFESSION CODE"},"desc":{"s":"UNKNOWN PROFESSION DESC"}}}', 'specialty': '{"m":{"code":{"s":"UNKNOWN SPECIALTY CODE"},"desc":{"s":"UNKNOWN SPECIALTY DESC"}}}'}}]

print(test == true)  # False
print(sorted(test) == sorted(true))  # False

But if I were to populate test and true using Python2,

I'd get

test2 = [{'item': {'flashInTargeting': '{"n":"1"}', 'npi': '{"s":"1467493197"}', 'name': '{"m":{"last":{"s":"Cernak"},"first":{"s":"Cynthia"}}}', 'flashVisible': '{"n":"1"}', 'lastModified': '{"n":"1234"}', 'profession': '{"m":{"code":{"s":"8"},"desc":{"s":"DESC"}}}', 'locations': '{"l":[{"m":{"geolocation":{"m":{"latitude":{"n":"42.054029999999997"},"longitude":{"n":"-87.888379999999998"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"5":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"2"},"formatted":{"s":"Address"},"vid":{"s":"123456781234561234"},"primary":{"s":"N"}}}}},{"m":{"geolocation":{"m":{"latitude":{"n":"42.567709999999998"},"longitude":{"n":"-87.930769999999995"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"5":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"1"},"formatted":{"s":"10105 74th St Ste 101 Atlanta WI 53142"},"vid":{"s":"243354971448280066"},"primary":{"s":"Y"}}}}}]}', 'specialty': '{"m":{"code":{"s":"POD"},"desc":{"s":"DESC"}}}', 'priority': '{"n":"0"}', 'pna': '{"s":"N"}', 'inboundTrx': '{"l":[{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"commTrxL1d":{"n":"0"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX"},"cashTrxL7d":{"n":"0"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PN"},"employeeName":{"s":"Test Rep"}}},{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"commTrxL1d":{"n":"1"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"1"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PN A"},"employeeName":{"s":"Test Rep"}}}]}', 'employeeId': '{"s":"15030"}'}}, {'item': {'flashInTargeting': '{"n":"0"}', 'npi': '{"s":"PDRP"}', 'name': '{"m":{"last":{"s":"PDRP"},"first":{"s":"PDRP"}}}', 'flashVisible': '{"n":"1"}', 'lastModified': '{"n":"1234"}', 'profession': '{"m":{"code":{"s":"UNKNOWN PROFESSION CODE"},"desc":{"s":"UNKNOWN PROFESSION DESC"}}}', 'locations': '{"l":[{"m":{"geolocation":{"m":{"latitude":{"n":"0"},"longitude":{"n":"0"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"00:00-00:00"},"5":{"s":"00:00-00:00"},"4":{"s":"00:00-00:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"99"},"formatted":{"s":"UNKNOWN ADDRESS"},"vid":{"s":"0"},"primary":{"s":"N"}}}}}]}', 'specialty': '{"m":{"code":{"s":"UNKNOWN SPECIALTY CODE"},"desc":{"s":"UNKNOWN SPECIALTY DESC"}}}', 'priority': '{"n":"0"}', 'pna': '{"s":"N"}', 'inboundTrx': '{"l":[{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"PDRP"},"commTrxL1d":{"n":"1"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"0"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PDRP"},"employeeName":{"s":"Test Rep"}}}]}', 'employeeId': '{"s":"15030"}'}}]

true2 = [{'item': {'flashInTargeting': '{"n":"1"}', 'npi': '{"s":"1467493197"}', 'employeeId': '{"s":"15030"}', 'flashVisible': '{"n":"1"}', 'lastModified': '{"n":"1234"}', 'profession': '{"m":{"code":{"s":"8"},"desc":{"s":"DESC"}}}', 'locations': '{"l":[{"m":{"geolocation":{"m":{"latitude":{"n":"42.054029999999997"},"longitude":{"n":"-87.888379999999998"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"5":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"2"},"formatted":{"s":"Address"},"vid":{"s":"123456781234561234"},"primary":{"s":"N"}}}}},{"m":{"geolocation":{"m":{"latitude":{"n":"42.567709999999998"},"longitude":{"n":"-87.930769999999995"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"08:00-17:00"},"5":{"s":"00:00-00:00"},"4":{"s":"09:00-17:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"1"},"formatted":{"s":"10105 74th St Ste 101 Atlanta WI 53142"},"vid":{"s":"243354971448280066"},"primary":{"s":"Y"}}}}}]}', 'specialty': '{"m":{"code":{"s":"POD"},"desc":{"s":"DESC"}}}', 'priority': '{"n":"0"}', 'pna': '{"s":"N"}', 'inboundTrx': '{"l":[{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"commTrxL1d":{"n":"0"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX"},"cashTrxL7d":{"n":"0"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PN"},"employeeName":{"s":"Test Rep"}}},{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"Atlanta"},"commTrxL1d":{"n":"1"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"1"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PN A"},"employeeName":{"s":"Test Rep"}}}]}', 'name': '{"m":{"last":{"s":"Cernak"},"first":{"s":"Cynthia"}}}'}}, {'item': {'flashInTargeting': '{"n":"0"}', 'npi': '{"s":"PDRP"}', 'employeeId': '{"s":"15030"}', 'flashVisible': '{"n":"1"}', 'lastModified': '{"n":"1234"}', 'profession': '{"m":{"code":{"s":"UNKNOWN PROFESSION CODE"},"desc":{"s":"UNKNOWN PROFESSION DESC"}}}', 'locations': '{"l":[{"m":{"geolocation":{"m":{"latitude":{"n":"0"},"longitude":{"n":"0"}}},"officeHours":{"m":{"1":{"s":"00:00-00:00"},"0":{"s":"00:00-00:00"},"3":{"s":"00:00-00:00"},"2":{"s":"00:00-00:00"},"5":{"s":"00:00-00:00"},"4":{"s":"00:00-00:00"},"6":{"s":"00:00-00:00"}}},"address":{"m":{"accessible":{"n":"1"},"ordinal":{"n":"99"},"formatted":{"s":"UNKNOWN ADDRESS"},"vid":{"s":"0"},"primary":{"s":"N"}}}}}]}', 'specialty': '{"m":{"code":{"s":"UNKNOWN SPECIALTY CODE"},"desc":{"s":"UNKNOWN SPECIALTY DESC"}}}', 'priority': '{"n":"0"}', 'pna': '{"s":"N"}', 'inboundTrx': '{"l":[{"m":{"reportDate":{"s":"2017-10-12"},"city":{"s":"PDRP"},"commTrxL1d":{"n":"1"},"district":{"s":"Test District"},"pharmacy":{"s":"SCRIPTS RX PHARMACY"},"cashTrxL7d":{"n":"0"},"commTrxL7d":{"n":"1"},"unfilledTrxL7d":{"n":"0"},"territory":{"s":"Test Territory"},"productName":{"s":"PDRP"},"employeeName":{"s":"Test Rep"}}}]}', 'name': '{"m":{"last":{"s":"PDRP"},"first":{"s":"PDRP"}}}'}}]


print test2 == true2  # True
print sorted(test2) == sorted(true2)  # True

I know test2 and test3 are the same because if I were to copy them to Diffchecker.com and click on Sort Lines, they're identical.

So essentially, test2 == true2 == test3 == true3 but in Python3, since the values are out of order, the comparison yields False. I tried all sorts of Dict sorting but I'm not sure why it's not working.

I'm populating these values using PySpark (Spark 2.4.2)

The expected result is for assert test3 == true3 to yield True

Can anyone help?

Can you please check your `print` statements in the first example for Python 3! I think there is something wrong. — AnsFourtyTwo, Sep 02 '19 at 21:16
You're right. It was a mistake done while trying to write the question but the problem still occurs — Roshin Jay, Sep 03 '19 at 01:39

Roshin Jay · Accepted Answer · 2019-09-03T01:48:47.370

The reason why they're showing as different in Python-3 is because the value of the key locationsin test has a different order when compared to true. Order matters here because the value is a string representation of nested dicts

However, in Python-2 the order for value of the key locations remains the same for both test and true

This is how I solved it using different and easier examples of test and true:

import json
from ast import literal_eval
from pprint import pformat

# Notice how the value for 'managedReps' are different in order for both test_list and true_list
test_list = [{'item': {'apiKey': '{"s":"testkey"}',
           'email': '{"s":"dcalabrese@mail.com"}',
           'employeeId': '{"s":"15025"}',
           'isManager': '{"n":"0"}',
           'lastModified': '{"n":"1234"}',
           'managedReps': '{"l":[{"m":{"last":{"s":"Calabrese"},"first":{"s":"Dave"},"employeeId":{"s":"15025"}}}]}',
           'name': '{"m":{"last":{"s":"Calabrese"},"first":{"s":"Dave"}}}',
           'orgId': '{"s":"1"}',
           'orgName': '{"s":"Org"}'}}]

true_list = [{'item': {'apiKey': '{"s":"testkey"}',
           'email': '{"s":"dcalabrese@mail.com"}',
           'employeeId': '{"s":"15025"}',
           'isManager': '{"n":"0"}',
           'lastModified': '{"n":"1234"}',
           'managedReps': '{"l":[{"m":{"last":{"s":"Calabrese"},"employeeId":{"s":"15025"},"first":{"s":"Dave"}}}]}',
           'name': '{"m":{"last":{"s":"Calabrese"},"first":{"s":"Dave"}}}',
           'orgId': '{"s":"1"}',
           'orgName': '{"s":"Org"}'}}]

def sort_list_of_dict(dataFrameList):
    sortedListofDict = []
    for dictionary in dataFrameList:
        sortedListofDict.append(order_dict(dictionary))

    return sorted(sortedListofDict, key=lambda k: k['item']['email'])  # Sort the list 

def order_dict(dictionary):
    resultDict = {}
    for k, v in sorted(dictionary.items()):
        if isinstance(v, dict):
            resultDict[k] = order_dict(v)
        elif isinstance(v, str):
            # Convert the str to a literal eval - in this case a dict
            v = literal_eval(v)  

            # Using pformat because it happens to sort nested dict somehow (Not the best way but it works)
            v = pformat(v).replace("\n", "").replace("  ", "")  

            # Convert the str to dict using eval to get the right representation of the 
            # dict and then using json.dumps to get the str representation
            resultDict[k] = json.dumps(literal_eval(v)) 

    return resultDict


assert sort_list_of_dict(test_list) == sort_list_of_dict(true_list)  # True

How to compare two list of nested dicts in python3 having string in value

1 Answers1