Python how to return a dictionary from a recursive function parsing a nested json?

Question

I'm trying to parse a nested json file for a specific key value "Date" and return a dictionary that should look like

{'Date', '2022-05-20T00:00:00Z'}

Here is my code:

meetingDate_dict = {}
def recursive_json(data,attr,m_dict):
    for k,v in data.items():
        if k == attr:
            for k2,v2 in v.items():
                m_dict = {attr, v2}
                print('IF: ',m_dict)
                return m_dict
        elif isinstance(v,dict):
            print('ELIF: ', k,v)
            return recursive_json(v,attr,m_dict)
        else:
            print('ELSE: ',k,v)

    print('END')

print('RETURN: ',recursive_json(assessment_json, "Date", meetingDate_dict))

Output:

ELSE:  $id 1
ELIF:  DataChangedEntry {'$id': '2', 'PathProperty': '/', 'Metadata': None, 'PreviousValue': None, 'CurrentValue': {'CosewicWsRefId': {'Value': 'QkNlrjq2HL9bhTQqU8-qH'}, 'Date': {'Value': '2022-05-20T00:00:00Z'}, 'YearSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'DateSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'Order': None, 'Type': {'Value': 'REGULAR'}, 'ReportType': {'Value': 'NEW'}, 'Stage': {'Value': 'ASSESSED'}, 'State': {'Value': 'PUBLISHED'}, 'StatusAndCriteria': {'Status': {'Value': 'EXTINCT'}, 'StatusComment': {'EnglishText': None, 'FrenchText': None}, 'StatusChange': {'Value': 'NOT_INITIALIZED'}, 'StatusCriteria': {'EnglishText': None, 'FrenchText': None}, 'ApplicabilityOfCriteria': {'ApplicabilityCriteriaList': []}}, 'Designation': None, 'Note': None, 'DomainEvents': [], 'Version': {'Value': 1651756761385.1248}, 'Id': {'Value': '3z3XlCkaXY9xinAbK5PrU'}, 'CreatedAt': {'Value': 1651756761384}, 'ModifiedAt': {'Value': 1651756785274}, 'CreatedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}, 'ModifiedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}}}
ELSE:  $id 2
ELSE:  PathProperty /
ELSE:  Metadata None
ELSE:  PreviousValue None
ELIF:  CurrentValue {'CosewicWsRefId': {'Value': 'QkNlrjq2HL9bhTQqU8-qH'}, 'Date': {'Value': '2022-05-20T00:00:00Z'}, 'YearSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'DateSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'Order': None, 'Type': {'Value': 'REGULAR'}, 'ReportType': {'Value': 'NEW'}, 'Stage': {'Value': 'ASSESSED'}, 'State': {'Value': 'PUBLISHED'}, 'StatusAndCriteria': {'Status': {'Value': 'EXTINCT'}, 'StatusComment': {'EnglishText': None, 'FrenchText': None}, 'StatusChange': {'Value': 'NOT_INITIALIZED'}, 'StatusCriteria': {'EnglishText': None, 'FrenchText': None}, 'ApplicabilityOfCriteria': {'ApplicabilityCriteriaList': []}}, 'Designation': None, 'Note': None, 'DomainEvents': [], 'Version': {'Value': 1651756761385.1248}, 'Id': {'Value': '3z3XlCkaXY9xinAbK5PrU'}, 'CreatedAt': {'Value': 1651756761384}, 'ModifiedAt': {'Value': 1651756785274}, 'CreatedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}, 'ModifiedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}}
ELIF:  CosewicWsRefId {'Value': 'QkNlrjq2HL9bhTQqU8-qH'}
ELSE:  Value QkNlrjq2HL9bhTQqU8-qH
END
RETURN:  None

So this way doesn't ever print the correct key:value pair, and returns None. If I remove the return statement in the elif I at least get the correct print out from the if statement, but I still get None returned:

meetingDate_dict = {}
def recursive_json(data,attr,m_dict):
    for k,v in data.items():
        if k == attr:
            for k2,v2 in v.items():
                m_dict = {attr, v2}
                print('IF: ',m_dict)
                return m_dict
        elif isinstance(v,dict):
            print('ELIF: ', k,v)
            recursive_json(v,attr,m_dict)
        else:
            print('ELSE: ',k,v)

    print('END')

print('RETURN: ',recursive_json(assessment_json, "Date", meetingDate_dict))

Output:

ELSE:  $id 1
ELIF:  DataChangedEntry {'$id': '2', 'PathProperty': '/', 'Metadata': None, 'PreviousValue': None, 'CurrentValue': {'CosewicWsRefId': {'Value': 'QkNlrjq2HL9bhTQqU8-qH'}, 'Date': {'Value': '2022-05-20T00:00:00Z'}, 'YearSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'DateSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'Order': None, 'Type': {'Value': 'REGULAR'}, 'ReportType': {'Value': 'NEW'}, 'Stage': {'Value': 'ASSESSED'}, 'State': {'Value': 'PUBLISHED'}, 'StatusAndCriteria': {'Status': {'Value': 'EXTINCT'}, 'StatusComment': {'EnglishText': None, 'FrenchText': None}, 'StatusChange': {'Value': 'NOT_INITIALIZED'}, 'StatusCriteria': {'EnglishText': None, 'FrenchText': None}, 'ApplicabilityOfCriteria': {'ApplicabilityCriteriaList': []}}, 'Designation': None, 'Note': None, 'DomainEvents': [], 'Version': {'Value': 1651756761385.1248}, 'Id': {'Value': '3z3XlCkaXY9xinAbK5PrU'}, 'CreatedAt': {'Value': 1651756761384}, 'ModifiedAt': {'Value': 1651756785274}, 'CreatedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}, 'ModifiedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}}}
ELSE:  $id 2
ELSE:  PathProperty /
ELSE:  Metadata None
ELSE:  PreviousValue None
ELIF:  CurrentValue {'CosewicWsRefId': {'Value': 'QkNlrjq2HL9bhTQqU8-qH'}, 'Date': {'Value': '2022-05-20T00:00:00Z'}, 'YearSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'DateSentToMinister': {'Value': '0001-01-01T00:00:00'}, 'Order': None, 'Type': {'Value': 'REGULAR'}, 'ReportType': {'Value': 'NEW'}, 'Stage': {'Value': 'ASSESSED'}, 'State': {'Value': 'PUBLISHED'}, 'StatusAndCriteria': {'Status': {'Value': 'EXTINCT'}, 'StatusComment': {'EnglishText': None, 'FrenchText': None}, 'StatusChange': {'Value': 'NOT_INITIALIZED'}, 'StatusCriteria': {'EnglishText': None, 'FrenchText': None}, 'ApplicabilityOfCriteria': {'ApplicabilityCriteriaList': []}}, 'Designation': None, 'Note': None, 'DomainEvents': [], 'Version': {'Value': 1651756761385.1248}, 'Id': {'Value': '3z3XlCkaXY9xinAbK5PrU'}, 'CreatedAt': {'Value': 1651756761384}, 'ModifiedAt': {'Value': 1651756785274}, 'CreatedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}, 'ModifiedBy': {'Value': 'Geordie.Powers@ec.gc.ca'}}
ELIF:  CosewicWsRefId {'Value': 'QkNlrjq2HL9bhTQqU8-qH'}
ELSE:  Value QkNlrjq2HL9bhTQqU8-qH
END
IF:  {'Date', '2022-05-20T00:00:00Z'}
END
ELSE:  EventAction Create
ELIF:  EventDataChange {'$ref': '2'}
ELSE:  $ref 2
END
ELSE:  CorrelationId 3z3XlCkaXY9xinAbK5PrU
ELSE:  EventId WGxlewsUAHayLHZ2LHvFk
ELSE:  EventTimeUtc 2022-05-06T13:15:31.7463355Z
ELSE:  EventDataVersion 1.0.0
ELSE:  EventType AssessmentCreatedInfrastructure
END
RETURN:  None

The json file:

{
  "$id": "1",
  "DataChangedEntry": {
    "$id": "2",
    "PathProperty": "/",
    "Metadata": null,
    "PreviousValue": null,
    "CurrentValue": {
      "CosewicWsRefId": {
        "Value": "QkNlrjq2HL9bhTQqU8-qH"
      },
      "Date": {
        "Value": "2022-05-20T00:00:00Z"
      },
      "YearSentToMinister": {
        "Value": "0001-01-01T00:00:00"
      },
      "DateSentToMinister": {
        "Value": "0001-01-01T00:00:00"
      },
      "Order": null,
      "Type": {
        "Value": "REGULAR"
      },
      "ReportType": {
        "Value": "NEW"
      },
      "Stage": {
        "Value": "ASSESSED"
      },
      "State": {
        "Value": "PUBLISHED"
      },
      "StatusAndCriteria": {
        "Status": {
          "Value": "EXTINCT"
        },
        "StatusComment": {
          "EnglishText": null,
          "FrenchText": null
        },
        "StatusChange": {
          "Value": "NOT_INITIALIZED"
        },
        "StatusCriteria": {
          "EnglishText": null,
          "FrenchText": null
        },
        "ApplicabilityOfCriteria": {
          "ApplicabilityCriteriaList": []
        }
      },
      "Designation": null,
      "Note": null,
      "DomainEvents": [],
      "Version": {
        "Value": 1651756761385.1248
      },
      "Id": {
        "Value": "3z3XlCkaXY9xinAbK5PrU"
      },
      "CreatedAt": {
        "Value": 1651756761384
      },
      "ModifiedAt": {
        "Value": 1651756785274
      },
      "CreatedBy": {
        "Value": "a@a"
      },
      "ModifiedBy": {
        "Value": "a@a"
      }
    }
  },
  "EventAction": "Create",
  "EventDataChange": {
    "$ref": "2"
  },
  "CorrelationId": "3z3XlCkaXY9xinAbK5PrU",
  "EventId": "WGxlewsUAHayLHZ2LHvFk",
  "EventTimeUtc": "2022-05-06T13:15:31.7463355Z",
  "EventDataVersion": "1.0.0",
  "EventType": "AssessmentCreatedInfrastructure"
}

I think you need to test the value returned by the recursive calls to see if they are non-null. If they are you can return that value, otherwise you need to keep processing. — Bobby Durrett, May 31 '22 at 23:51
Might help to show the Python code that loads assessment_json — Bobby Durrett, May 31 '22 at 23:54
@BobbyDurrett the return recursive is the ```ELIF: ...``` print out. I can print the value I want in the function, I just can't return it outside the function. — Mitch, Jun 01 '22 at 00:39
This question appears to be nearly identical to this one https://stackoverflow.com/questions/72455526/how-to-return-specific-key-value-pair-from-nested-json-without-knowing-location/72455622 for which you received an answer you accepted - why shouldn't this question be closed, referring to the previous one? — Grismar, Jun 01 '22 at 05:11

Nguyễn Minh Hiếu · Accepted Answer · 2022-06-02T02:52:21.663

I think the problem here is because when looking for attr in deeper levels, you didn't check whether recursive_json(v,attr,m_dict) is returning the actual desired value or None and just return it anyway. Changing the code to:

def recursive_json(data,attr,m_dict):
    ...
       # find attr value in deeper levels
       elif isinstance(v,dict):
           result = recursive_json(v,attr,m_dict)
           # if result is not None, return it
           if result:
               return result
    ...

I would refactor your recursive_json function as follows, removing m_dict as it seems unnecessary:

def recursive_json(data: dict, lookup_key: str, value_key: str = "Value"):
    # look for all key-value pairs at current levels
    for k, v in data.items():
        # Check if current pair satisfies the following conditions:
        # - key equal to lookup key
        # - value is a dictionary
        # - value dictionary has subkey named `Value`
        if k == lookup_key and isinstance(v, dict) and value_key in v:
            return {lookup_key: v.get(value_key)}
        
        # else if value is a dictionary, search `lookup_key` in deeper levels
        elif isinstance(v, dict):
            lookup_result = recursive_json(v, lookup_key, value_key)
            
            # if value for `lookup_key` is found in deeper levels, return the value
            if lookup_result:
                return lookup_result
    
    # return None as no satisfied value found
    return None

Edit: For multiple instances of lookup_key, try the following method:

def recursive_json(data: dict, lookup_keys: list, value_key: str = "Value"):
    if isinstance(lookup_keys, str):
        lookup_keys = [lookup_keys]

    # BFS
    search_q = [data]
    # look for all key-value pairs at current levels
    lookup_result = {}
    while search_q and lookup_keys:
        in_search_dict = search_q.pop(0)
        for k, v in in_search_dict.items():
            # Check if current pair satisfies the following conditions:
            # - key equal to lookup key
            # - value is a dictionary
            # - value dictionary has subkey named `Value`
            if isinstance(v, dict):
                if value_key in v and k in lookup_keys:
                    # remove `value_key` from current dict
                    value = v.pop(value_key)
                    # update lookup result
                    lookup_result.update({k: value})
                    # remove found key from lookup keys
                    lookup_keys.remove(k)

                    # if all lookup keys are found, break the while loop
                    if not lookup_keys:
                        break
                if v:
                    search_q.append(v)

    # return lookup result
    return lookup_result

print(recursive_json(assessment_json, ["Date", "ModifiedBy", "CreatedBy"]))
# {'Date': '2022-05-20T00:00:00Z', 'CreatedBy': 'a@a', 'ModifiedBy': 'a@a'}

Thank you this works. Would this work for multiple instances of the ```lookup_key``` as well? — Mitch, Jun 01 '22 at 15:58
I have edited my answer to add multiple lookup key support. Check it out if it suits your needs. — Nguyễn Minh Hiếu, Jun 02 '22 at 02:53

Python how to return a dictionary from a recursive function parsing a nested json?

1 Answers1