I have around 30k license numbers that I want to search from a website and extract all the relevant information from it When I tried the extracting the information from the function below by looping through multiple license_nums the code works fine and gives me what I am looking for
# create a UserAgent object to generate random user agents
user_agent = UserAgent()
# create a ChromeOptions object to set the user agent in the browser header
chrome_options = Options()
chrome_options.add_argument(f'user-agent={user_agent.random}')
chrome_options.add_argument("start-maximized")
# create a webdriver instance with the ChromeOptions object
driver = webdriver.Chrome(options=chrome_options,executable_path=r'C:\WebDrivers\ChromeDriver\chromedriver_win32\chromedriver.exe')
driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
driver.execute_cdp_cmd('Network.setUserAgentOverride', {"userAgent": 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.53 Safari/537.36'})
print(driver.execute_script("return navigator.userAgent;"))
form_url = "https://cdicloud.insurance.ca.gov/cal/LicenseNumberSearch?handler=Search"
driver.get(form_url)
license_num = ['0726675', '0747600', '0691046', '0D95524', '0E77989', '0L78427']
def get_license_info(license):
if license not in license_num:
return pd.DataFrame()
df_license = []
search_box = driver.find_element('id','SearchLicenseNumber').send_keys(license)
time.sleep(randint(15,100))
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "btnSearch"))).click()
page_source = driver.page_source
soup = BeautifulSoup(page_source, "html.parser")
table = soup.find('table', id='searchResult')
license_name = []
license_number =[]
#extract all license names on the page
# Collecting Ddata
for row in table.tbody.find_all('tr'):
# Find all data for each column
columns = row.find_all('td')
if(columns != []):
l_name = columns[0].text.strip().replace("\t"," ")
license_name.append(l_name)
license_number.append(columns[1].text.strip())
print(l_name)
for row in range(0, len(license_name)):
first_page_handle = driver.current_window_handle
time.sleep(5)
WebDriverWait(driver, 40).until(EC.element_to_be_clickable((By.XPATH, f"//table[@id='searchResult']/tbody/tr[{row+1}]/td[2]/a"))).click()
try:
driver.switch_to.window(driver.window_handles[1])
html = driver.page_source
soup = BeautifulSoup(html, "lxml")
#Grab license type and Expiration date
table_l = soup.find('table', id='licenseDetailGrid')
data = []
for tr in table_l.find_all('tr'):
row = [td.text for td in tr.find_all('td')]
data.append(row)
df1 = pd.DataFrame(data, columns=['license_type','original_issue_date','status','status_date','exp_date'])
time.sleep(5)
business = soup.find("div",id="collapse-LicenseDetailSection").extract()
b_list = list(business.stripped_strings)
df_final = df1[df1['license_type'].str.contains("Accident",na=False)]
df_final = df_final.assign(license_type=df_final['license_type'].str.extract('(.*)\n'))
df_final['license_name'] = l_name
df_final['license_number'] = license
df_license.append(df_final)
driver.close()
driver.switch_to.window(first_page_handle)
except NoSuchWindowException:
print("Window closed, skipping to next license")
driver.find_element('id','SearchLicenseNumber').clear()
time.sleep(5)
return pd.concat(df_license)
when I try to put it run with multi thread it doesn't show the value in the search field and throws error
approach 1 from (Scraping multiple webpages at once with Selenium)
Error: It runs for first item in the license_num_list and then start seraching for empty license number and throws 'An exception occurred: 'NoneType' object has no attribute 'tbody'
with futures.ThreadPoolExecutor() as executor:
# store the url for each thread as a dict, so we can know which thread fails
future_results = {license: executor.submit(get_license_info, license) for license in license_num}
for license, future in future_results.items():
try:
df_license = pd.concat([f.result() for f in future_results.values()])
except Exception as exc:
print('An exception occurred: {}'.format(exc))
approach 2 from (How to run `selenium-chromedriver` in multiple threads)
Error: Even this approach only search for first item in the list and throws 'Message: stale element reference: element is not attached to the page document'
start_time = time.time()
threads = []
for license in license_num: # each thread could be like a new 'click'
th = threading.Thread(target=get_license_info, args=(license,))
th.start() # could `time.sleep` between 'clicks' to see whats'up without headless option
threads.append(th)
for th in threads:
th.join() # Main thread wait for threads finish
print("multiple threads took ", (time.time() - start_time), " seconds")
Can anybody help me with this. Thank you in advance
