I have a dataframe with the following schema:
I try to fetch all the data from this dataframe. I use df.collect() method to iterate through the entire dataframe and then pulling the values out of the columns one-by-one. But it seems like its not iterating through the entire tree and just pulling through the initial parent row only.
def parseCol(landing_df,data):
for i in landing_df.collect():
parent_id = i["parent_id"]
shared = "null"
if (len(i["children"]))>1:
# print(len(i["children"]))
# if(len(i["children"])>1):
data.append([i["project_id"],i["id"],i["name"],i["order"],i["pid"],i["created_date"],i["last_modified_date"], str(parent_id),i["description"],i["recursive"],i["links"][0][0],str(shared)])
for j in i["children"]:
if(('shared') not in (i)):
shared = 'null'
else:
shared = i['shared']
if(('project_id') not in (j)):
project_id = "null"
else:
project_id = j['project_id']
data.append([project_id,j["id"],j["name"],j["order"],j["pid"],j["created_date"],j["last_modified_date"],str(j["parent_id"]),j["description"],j["recursive"],j["links"][0][0],str(shared)])
# print(-1)
elif(len(i["children"])==0):
data.append([i["project_id"],i["id"],i["name"],i["order"],i["pid"],i["created_date"],i["last_modified_date"],"null",i["description"],i["recursive"],i["links"][0][0],str(shared)])
return data
Can someone suggest some better way to do this.
