"errorMessage": "dictionary keys changed during iteration"

Question

I am new to python. I am using this lambda that will change CTR data to parquet form upon receiving in S3 for QuickSight Visualization (using Athen and AWS Glue)

here is the Code

from __future__ import print_function
import json
import urllib
import boto3
import os
import re
import copy

s3 = boto3.resource('s3')
glue = boto3.client('glue')
client = boto3.client('s3')
def convertColumntoLowwerCaps(obj):
    for key in obj.keys():
        new_key = re.sub(r'[\W]+', '', key.lower())
        v = obj[key]
        if isinstance(v, dict):
            if len(v) > 0: 
                convertColumntoLowwerCaps(v)
        if new_key != key:
            obj[new_key] = obj[key]
            del obj[key]
    return obj

def lambda_handler(event, context):
    print("Event :", event)
    bucket = event['Records'][0]['s3']['bucket']['name']
    # key = urllib.parse.unquote_plus(event['Records'][0]['s3']['object']['key'].encode('utf8'))
    key = event["Records"][0]["s3"]["object"]["key"].encode('utf8')
    key = key.decode('UTF-8')
    try:
        client.download_file(bucket, key, "/tmp/file.json")        
        with open('/tmp/out.json', 'w') as output, open("/tmp/file.json", "r") as file:
            i = 0
           # line_copy = line
            for line in file:  
                # split_line = tmp.split(b';')
                
                for object in line.replace("}{","}\n{").split("\n"):
                    record = json.loads(object,object_hook=convertColumntoLowwerCaps)
                    if i != 0:
                        output.write("\n")
                    output.write(json.dumps(record))
                    i += 1
        newkey = 'flatfiles/' + key.replace("/", "")                
        client.upload_file('/tmp/out.json', bucket.name, newkey)
        s3.Object(bucket,key).delete()
        return "success"
    except Exception as e:
        print(e)
        print('Error coping object {} from bucket {}'.format(key, bucket))
        raise e

here is the Error Message,

{ "errorMessage": "dictionary keys changed during iteration",

"errorType": "RuntimeError",

"requestId": "f511c16e-9a6c-4c16-92e3-da47a5a29143",

"stackTrace": [ " File "/var/task/index.py", line 51, in lambda_handler\n raise e\n",

"  File \"/var/task/index.py\", line 39, in lambda_handler\n    record =

json.loads(object,object_hook=convertColumntoLowwerCaps)\n", " File "/var/lang/lib/python3.9/json/init.py", line 359, in loads\n return cls(**kw).decode(s)\n", " File "/var/lang/lib/python3.9/json/decoder.py", line 337, in decode\n obj, end = self.raw_decode(s, idx=_w(s, 0).end())\n", " File "/var/lang/lib/python3.9/json/decoder.py", line 353, in raw_decode\n obj, end = self.scan_once(s, idx)\n", " File "/var/task/index.py", line 13, in convertColumntoLowwerCaps\n for key in obj.keys():\n" ] } Any kind of help will be appreciated.

see https://stackoverflow.com/questions/45877614/how-to-change-all-the-dictionary-keys-in-a-for-loop-with-d-items — balderman, Apr 24 '22 at 06:57

score 0 · Answer 1 · 2022-04-24T07:08:53.203

The cause of the error is being highlighted here File "/var/task/index.py", line 13, in convertColumntoLowwerCaps\n for key in obj.keys():\n"

You are iterating over the keys of obj and then changing obj. Try changing for key in obj.keys(): to for key in list(obj.keys()):

Calling list creates a new list from the keys. You will be iterating over the list instead of the keys of obj.

"errorMessage": "dictionary keys changed during iteration"

1 Answers1