I'm trying to write a webscraper to automate document download. The button is buried within multiple shadow roots. I suspect the issue lies with my webdriver's current loaded page. I'm fairly certain I need to use JavaScript to drill through the shadow-roots, but I'm not sure.
I keep getting this error: JavascriptException: Message: javascript error: Cannot read properties of null (reading 'shadowRoot')
Here's my complete code:
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
options.add_experimental_option("excludeSwitches", ["enable-automation"])
options.add_experimental_option('useAutomationExtension', False)
from bs4 import BeautifulSoup
import requests
import time
import pandas as pd
import numpy as np
driver = webdriver.Chrome(options=options)
url = "http://oris.co.palm-beach.fl.us/or_web1/or_sch_1.asp"
driver.get(url)
driver.find_element_by_xpath('//*[@id="window1"]/table/tbody/tr/td/table/tbody/tr[1]/td[3]/a').click()
doc_type = driver.find_element_by_xpath('//*[@id="window2"]/table/tbody/tr/td/table/tbody/tr[3]/td/table/tbody/tr[1]/td/form/font[2]/input')
doc_type.send_keys('D')
consideration_input = driver.find_element_by_name('consideration')
consideration_input.send_keys('5,000,000')
from_date = driver.find_element_by_xpath('//*[@id="window2"]/table/tbody/tr/td/table/tbody/tr[3]/td/table/tbody/tr[1]/td/form/table/tbody/tr/td[1]/font[3]/input')
from_date.clear()
from_date.send_keys('11/04/2021')
to_date = driver.find_element_by_xpath('//*[@id="window2"]/table/tbody/tr/td/table/tbody/tr[3]/td/table/tbody/tr[1]/td/form/table/tbody/tr/td[1]/font[5]/input')
to_date.clear()
to_date.send_keys('11/05/2021')
submit_button = driver.find_element_by_xpath('//*[@id="window2"]/table/tbody/tr/td/table/tbody/tr[3]/td/table/tbody/tr[1]/td/form/input[2]')
submit_button.click()
url_list = driver.find_elements_by_class_name("list_2[href]")
url_list[0].click()
driver.switch_to.frame('FrmDes')
image_button = driver.find_element_by_xpath("/html/body/table[2]/tbody/tr/td[2]/map[4]/area")
image_button.click()
driver.switch_to.default_content()
driver.switch_to.frame('FrmImg')
view_pdf = driver.find_element_by_xpath('/html/body/table[2]/tbody/tr[1]/td[1]/form/input[12]')
view_pdf.click()
try:
driver.page_source
except Exception as e:
print(e)
element = driver.execute_script(
"return document.querySelector('pdf-viewer').shadowRoot.querySelector('viewer-toolbar').shadowRoot.querySelector('viewer-download-controls')"
)
