-2

I have three lists of dicts, some sample data can look like:

list_1.append({"shared_id": "111", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 2}]})
list_1.append({"shared_id": "222", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 3}]})

list_2.append({"shared_id": "111", "users": [{"id": "444", "clicks": 1}, {"id": "555", "clicks": 2}]})
list_2.append({"shared_id": "222", "users": [{"id": "444", "clicks": 3}, {"id": "555", "clicks": 3}]})

list_3.append({"shared_id": "111", "users": [{"id": "444", "bounces": 2}, {"id": "555", "bounces": 1}]})
list_3.append({"shared_id": "222", "users": [{"id": "444", "bounces": 3}, {"id": "555", "bounces": 2}]})

Using the shared id and ids on the users I want to create 1 combined list that looks like:

combined_list.append([{"shared_id": "111", "users": [{"id": "444", "opens": 2, "clicks": 1, "bounces": 2, "combined_id": 444111}, {"id": "555", "opens": 3, "clicks": 2, "bounces": 1, "combined_id": 555111}}])

combined_list.append([{"shared_id": "222", "users": [{"id": "444", "opens": 2, "clicks": 3, "bounces": 1, "combined_id": 444222}, {"id": "555", "opens": 3, "clicks": 3, "bounces": 2, "combined_id": 555222}}])

I have solved it when I had 2 lists using nested for loops, but the code is super messy and does not feel very pythonized, is there a better way now that I am using 3 lists?

for o in list_1:
        for c in list_2:
            if o['shared_id'] == c['shared_id']:
                csd = {
                    'users': [],
                    'shared_id': o['shared_id']
                }
                for op in o['users']:
                    for cp in c['users']:
                        if op['id'] == cp['id']:
                            cpd = {
                                'opens': op['opens'],
                                'clicks': cp['clicks'],
                                'combined_id': '{}{}'.format(
                                    op['id'],
                                    csd['shared_id']
                                    )
                            }

                            csd['users'].append(cpd)

                combined.append(csd)
Majs
  • 607
  • 2
  • 7
  • 20
  • Can you show the code you are using? – Dschoni Aug 02 '19 at 09:08
  • Your code and desired output do not match. Do you need that `'combined_id'`? The "best" solution also depends on details. You might want to input data into a pandas dataframe first if you do anything else with it besides comvining lists. – IcedLance Aug 02 '19 at 09:26
  • @IcedLance Ah good catch, sorry. I do need it yes, will add it to my edit. – Majs Aug 02 '19 at 09:27
  • For simple solution you can move you code to a separate function `def combine(a,b)` but replace `cpd=•••` part with `cpd={**op, **cp, 'combined_id': xx+x}` . But that's just a shortcut, it replaces duplicate entries (but you make sure they are the same. If you want anything more like sum the values with same key you shoyld look into pandas. – IcedLance Aug 02 '19 at 09:44

5 Answers5

1

You could use itertools.groupby (doc) for grouping elements:

list_1, list_2, list_3 = [], [], []

list_1.append({"shared_id": "111", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 2}]})
list_1.append({"shared_id": "222", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 3}]})

list_2.append({"shared_id": "111", "users": [{"id": "444", "clicks": 1}, {"id": "555", "clicks": 2}]})
list_2.append({"shared_id": "222", "users": [{"id": "444", "clicks": 3}, {"id": "555", "clicks": 3}]})

list_3.append({"shared_id": "111", "users": [{"id": "444", "bounces": 2}, {"id": "555", "bounces": 1}]})
list_3.append({"shared_id": "222", "users": [{"id": "444", "bounces": 3}, {"id": "555", "bounces": 2}]})

from itertools import groupby

shared_ids = {}
for v, g in groupby(sorted(list_1 + list_2 + list_3, key=lambda k: k['shared_id']), lambda k: k['shared_id']):
    for shared_id in g:
        for user in shared_id['users']:
            shared_ids.setdefault(v, {}).setdefault(user['id'], {}).update(**user)
            shared_ids[v][user['id']]['combined_id'] = '{}{}'.format(user['id'], v)


out = [{'shared_id': k, 'users': [shared_ids[k][kk] for kk in shared_ids[k]]} for k in shared_ids]

from pprint import pprint
pprint(out)

Prints:

[{'shared_id': '111',
  'users': [{'bounces': 2, 'clicks': 1, 'combined_id': '444111', 'id': '444', 'opens': 2},
            {'bounces': 1, 'clicks': 2, 'combined_id': '555111', 'id': '555', 'opens': 2}]},
 {'shared_id': '222',
  'users': [{'bounces': 3, 'clicks': 3, 'combined_id': '444222', 'id': '444', 'opens': 2},
            {'bounces': 2, 'clicks': 3, 'combined_id': '555222', 'id': '555', 'opens': 3}]}]

NOTE: If you print shared_ids variable, you get:

{'111': {'444': {'bounces': 2, 'clicks': 1, 'combined_id': '444111', 'id': '444', 'opens': 2},
         '555': {'bounces': 1, 'clicks': 2, 'combined_id': '555111', 'id': '555', 'opens': 2}},
 '222': {'444': {'bounces': 3, 'clicks': 3, 'combined_id': '444222', 'id': '444', 'opens': 2},
         '555': {'bounces': 2, 'clicks': 3, 'combined_id': '555222', 'id': '555', 'opens': 3}}}

Maybe this dictionary will be better to manipulate further.

Andrej Kesely
  • 168,389
  • 15
  • 48
  • 91
1

Here a simple function for that:

from pprint import pprint

def combine_lists(*lists):
    result = {}
    # For each list
    for lst in lists:
        # For each dict in the list
        for d1 in lst:
            # Get or make common dict for shared_id
            shared_id = d1["shared_id"]
            shared_dict = result.setdefault(shared_id, {})
            # For each user dict in the dict
            for d2 in d1["users"]:
                # Get or make user dict
                user_id = d2["id"]
                if user_id not in shared_dict:
                    shared_dict[user_id] = {"combined_id": "{}{}".format(user_id, shared_id)}
                # Update information in user dictionary
                shared_dict[user_id].update(d2)
    # Make output as a list
    return [{"shared_id": k, "users": list(v.values())} for k, v in result.items()]

# Test
list_1 = []
list_1.append({"shared_id": "111", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 2}]})
list_1.append({"shared_id": "222", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 3}]})
list_2 = []
list_2.append({"shared_id": "111", "users": [{"id": "444", "clicks": 1}, {"id": "555", "clicks": 2}]})
list_2.append({"shared_id": "222", "users": [{"id": "444", "clicks": 3}, {"id": "555", "clicks": 3}]})
list_3 = []
list_3.append({"shared_id": "111", "users": [{"id": "444", "bounces": 2}, {"id": "555", "bounces": 1}]})
list_3.append({"shared_id": "222", "users": [{"id": "444", "bounces": 3}, {"id": "555", "bounces": 2}]})

pprint(combine_lists(list_1, list_2, list_3))
# [{'shared_id': '111',
#   'users': [{'bounces': 2,
#              'clicks': 1,
#              'combined_id': '444111',
#              'id': '444',
#              'opens': 2},
#             {'bounces': 1,
#              'clicks': 2,
#              'combined_id': '555111',
#              'id': '555',
#              'opens': 2}]},
#  {'shared_id': '222',
#   'users': [{'bounces': 3,
#              'clicks': 3,
#              'combined_id': '444222',
#              'id': '444',
#              'opens': 2},
#             {'bounces': 2,
#              'clicks': 3,
#              'combined_id': '555222',
#              'id': '555',
#              'opens': 3}]}]
jdehesa
  • 58,456
  • 7
  • 77
  • 121
1

You could use itertools.groupby() to group the ids together, then merge the dictionaries as needed:

from collections import ChainMap
from itertools import chain, groupby
from operator import itemgetter

combined_list = []
for k, g in groupby(sorted(chain(list_1, list_2, list_3), key=itemgetter('shared_id')), key=itemgetter('shared_id')):
    users = []
    for k2, g2 in groupby(sorted(chain(*map(itemgetter('users'), g)), key=itemgetter('id')), key=itemgetter('id')):
        users.append({'id': k2, 'combined_id': k2 + k, **ChainMap(*g2)})

    combined_list.append({'shared_id': k, 'users': users})
print(combined_list)

Output:

[{'shared_id': '111', 'users': [{'id': '444', 'combined_id': '444111', 'bounces': 2, 'clicks': 1, 'opens': 2}, {'id': '555', 'combined_id': '555111', 'bounces': 1, 'clicks': 2, 'opens': 2}]}, {'shared_id': '222', 'users': [{'id': '444', 'combined_id': '444222', 'bounces': 3, 'clicks': 3, 'opens': 2}, {'id': '555', 'combined_id': '555222', 'bounces': 2, 'clicks': 3, 'opens': 3}]}]
Eugene Yarmash
  • 142,882
  • 41
  • 325
  • 378
1

Objected Oriented Solution

# Data structure to manage shared object information
class shared(object):
  def __init__(self, i):
    self.i = i
    self.users = dict()

  # Add information of the particulate user
  def update_user(self, user_id, key, value):
    if user_id in self.users:
      self.users[user_id][key] = value
    else:
      self.users[user_id]= {key:value,
                            "id":user_id, 
                            "combined_id": "{0}{1}".format(user_id,self.i)}      

  # Return the information is required format
  def get(self):
    return  {
        'shared_id': self.i,
        'users' : [ user for user in self.users.values()]
    }

Data

list_1 = []
list_2 = []
list_3 = []

list_1.append({"shared_id": "111", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 2}]})
list_1.append({"shared_id": "222", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 3}]})

list_2.append({"shared_id": "111", "users": [{"id": "444", "clicks": 1}, {"id": "555", "clicks": 2}]})
list_2.append({"shared_id": "222", "users": [{"id": "444", "clicks": 3}, {"id": "555", "clicks": 3}]})

list_3.append({"shared_id": "111", "users": [{"id": "444", "bounces": 2}, {"id": "555", "bounces": 1}]})
list_3.append({"shared_id": "222", "users": [{"id": "444", "bounces": 3}, {"id": "555", "bounces": 2}]})

Parse the Data

info = {}
for l in list_1+list_2+list_3:
  # Is there already information for this shared_id ?
  if l["shared_id"] not in info:
    info[l["shared_id"]] = shared(l["shared_id"])
  # Parse all the users  
  for user_info in l['users']:
    user_id = user_info["id"]
    # Parse the user information
    for key,value in user_info.items():
      if key != "id":
        # Update the user information
        info[l["shared_id"]].update_user(user_id, key, value)


result = [x.get() for x in info.values()]
print (result)

Output

[{'shared_id': '111', 'users': [{'opens': 2, 'id': '444', 'combined_id': '444111', 'clicks': 1, 'bounces': 2}, {'opens': 2, 'id': '555', 'combined_id': '555111', 'clicks': 2, 'bounces': 1}]}, 
 {'shared_id': '222', 'users': [{'opens': 2, 'id': '444', 'combined_id': '444222', 'clicks': 3, 'bounces': 3}, {'opens': 3, 'id': '555', 'combined_id': '555222', 'clicks': 3, 'bounces': 2}]}]
mujjiga
  • 16,186
  • 2
  • 33
  • 51
0
global_list = []

def add_item(item):
    item_found = False
    for e_item in global_list:
        if e_item['shared_id'] == item['shared_id']:
            item_found = True
            e_users = e_item['users']
            users = item['users']
            for user in users:
                user_found = False
                for e_user in e_users:
                    if e_user['id'] == user['id']:
                        user_found = True
                        e_user.update(user)
                if user_found is False:
                    e_users.append(user)
    if item_found is False:
        global_list.append(item)    



list_1 = []
list_2 = []
list_3 = []

list_1.append({"shared_id": "111", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 2}]})
list_1.append({"shared_id": "222", "users": [{"id": "444", "opens": 2}, {"id": "555", "opens": 3}]})

list_2.append({"shared_id": "111", "users": [{"id": "444", "clicks": 1}, {"id": "555", "clicks": 2}]})
list_2.append({"shared_id": "222", "users": [{"id": "444", "clicks": 3}, {"id": "555", "clicks": 3}]})

list_3.append({"shared_id": "111", "users": [{"id": "444", "bounces": 2}, {"id": "555", "bounces": 1}]})
list_3.append({"shared_id": "222", "users": [{"id": "444", "bounces": 3}, {"id": "555", "bounces": 2}]})

for item in list_1+list_2+list_3:
    add_item(item)
print(global_list)
Asif Mohammed
  • 1,323
  • 1
  • 15
  • 29