To get the columns of the table also the first column containing link by clicking that link to get the data

Question

I have the below link

http://www.igrmaharashtra.gov.in/eASR/eASRCommon.aspx?hDistName=Pune

In this i want to scrape data in proper format in excel.The SurveyNo link contains the data when it is click i want the row-wise data with the data on clicking the survey number.

Also want the format that i have attached in the image (desired output in excel)

import urllib.request
from bs4 import BeautifulSoup
import csv
import os
from selenium import webdriver
from selenium.webdriver.support.select import Select
from selenium.webdriver.common.keys import Keys
import time
url = 'http://www.igrmaharashtra.gov.in/eASR/eASRCommon.aspx? 
hDistName=Pune'
chrome_path =r'C:/Users/User/AppData/Local/Programs/Python/Python36/Scripts/chromedriver.exe'
driver = webdriver.Chrome(executable_path=chrome_path)
driver.implicitly_wait(10)
driver.get(url)
Select(driver.find_element_by_name('ctl00$ContentPlaceHolder5$ddlTaluka')).select_by_value('5')
Select(driver.find_element_by_name('ctl00$ContentPlaceHolder5$ddlVillage')).select_by_value('1872')
soup=BeautifulSoup(driver.page_source, 'lxml')
table = soup.find("table" , attrs = {'id':'ctl00_ContentPlaceHolder5_grdUrbanSubZoneWiseRate' })
with open('Baner.csv', 'w',encoding='utf-16',newline='') as csvfile:
     f = csv.writer(csvfile, dialect='excel')
     f.writerow(['SurveyNo','Subdivision', 'Open ground', 'Resident house','Offices','Shops','Industrial','Unit (Rs./)'])  # headers
     rows = table.find_all('tr')[1:] 
     data=[]
     for tr in rows:  
         cols = tr.find_all('td')
         for td in cols:
              links = driver.find_elements_by_link_text('SurveyNo')
              l =len(links)
              data12 =[]
              for i in range(l):
                   newlinks = driver.find_elements_by_link_text('SurveyNo')
                   newlinks[i].click()
                   soup = BeautifulSoup(driver.page_source, 'lxml')
                   td1 = soup.find("textarea", attrs={'class': 'textbox'})
                   data12.append(td1.text)
                   data.append(td.text)
                   data.append(data12)
              print(data)

Please find the image. In that format I required the output of scrape data.

Answer 1

You could do the following and simply re-arrange columns at end along with desired renaming. There is the assumption SurveyNo exists for all wanted rows. I extract the hrefs from the SurveyNo cells which are actually executable strings you can pass to execute_script to show the survey numbers without worrying about stale element etc....

from selenium import webdriver
import pandas as pd

url = 'http://www.igrmaharashtra.gov.in/eASR/eASRCommon.aspx?hDistName=Pune'
d = webdriver.Chrome()
d.get(url)
d.find_element_by_css_selector('[value="5"]').click()
d.find_element_by_css_selector('[value="1872"]').click()
tableElement = d.find_element_by_id('ctl00_ContentPlaceHolder5_grdUrbanSubZoneWiseRate')
table = pd.read_html(tableElement.get_attribute('outerHTML'))[0]
table.columns = table.iloc[0]
table = table.iloc[1:]
table = table[table.Select == 'SurveyNo'] #assumption SurveyNo exists for all wanted rows
surveyNo_scripts = [item.get_attribute('href') for item in d.find_elements_by_css_selector("#ctl00_ContentPlaceHolder5_grdUrbanSubZoneWiseRate [href*='Select$']")]
i = 0
for script in surveyNo_scripts:
    d.execute_script(script)
    surveys = d.find_element_by_css_selector('textarea').text
    table.iloc[i]['Select'] = surveys
    i+=1   
print(table)
#rename and re-order columns as required
table.to_csv(r"C:\Users\User\Desktop\Data.csv", sep=',', encoding='utf-8-sig',index = False ) 

---

Output before rename and re-order:

---

In a loop you can concat all dfs and then write out in one go (my preference - shown here) or later append as shown here