## RJ: import libraries ##
## RJ: END OF COMMENT ##
from urllib.request import urlopen
from bs4 import BeautifulSoup
import csv
import time
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.support.ui import Select
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
##RJ specify the url
quote_page = 'https://www.onlineprinters.fr/p/autocollants-offset-a8'
##RJ query the website and return the html to the variable 'page'
page = urlopen(quote_page)
##RJ parse the html using beautiful soup and store in variable `soup`
soup = BeautifulSoup(page, 'html.parser')
driver = webdriver.Chrome('C:\\Users\\rashm\\Desktop\\WebScraper\\chromedriver.exe')
driver.get("https://www.onlineprinters.fr/p/autocollants-offset-a8")
inputPapier = driver.find_element_by_class_name('wsmds_input')
cookiepopup = driver.find_element_by_id('ws_cookie_layer_button')
## RJ: Using a series of actions ##
actions = ActionChains(driver)
actions.click(cookiepopup)
actions.click(inputPapier)
actions.perform()
selectPaper = Select(driver.find_element_by_name('input_var_PAKA840_1_1')).options
driver.execute_script("document.getElementsByName('input_var_PAKA840_1_1')[0].style.display = 'block';")
optionsPaper=[]
for option in selectPaper:
optionsPaper.append(option.text)
selectTirage = Select(driver.find_element_by_name('input_var_PAKA840_2_1')).options
driver.execute_script("document.getElementsByName('input_var_PAKA840_2_1')[0].style.display = 'block';")
optionsTirage=[]
for option in selectTirage:
optionsTirage.append(option.text)
selectDelai = Select(driver.find_element_by_name('input_var_ZAKA834Y_1_3')).options
driver.execute_script("document.getElementsByName('input_var_ZAKA834Y_1_3')[0].style.display = 'block';")
optionsDelai=[]
for option in selectDelai:
optionsDelai.append(option.text)
selectVerification = Select(driver.find_element_by_name('input_var_ZAKXXXXD_1_2')).options
driver.execute_script("document.getElementsByName('input_var_ZAKXXXXD_1_2')[0].style.display = 'block';")
optionsVerification=[]
for option in selectVerification:
optionsVerification.append(option.text)
bacsicPriceBox = driver.find_element_by_id('pr_basispreis')
for indexPaper in range(0, len(optionsPaper)):
print('---------------------TESTING FOR--------------------')
driver.execute_script("document.getElementsByName('input_var_PAKA840_1_1')[0].style.display = 'block';")
paperElement = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, "input_var_PAKA840_1_1")))
Select(paperElement).select_by_index(indexPaper)
for indexTirage in range(0, len(optionsTirage)):
driver.execute_script("document.getElementsByName('input_var_PAKA840_2_1')[0].style.display = 'block';")
tirageElement = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, "input_var_PAKA840_2_1")))
Select(tirageElement).select_by_index(indexTirage)
print('---------------------TESTING TIRAGE SELECTED--------------------'+str(indexTirage))
for indexDelai in range(0, len(optionsDelai)):
driver.execute_script("document.getElementsByName('input_var_ZAKA834Y_1_3')[0].style.display = 'block';")
delaiElement = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, "input_var_ZAKA834Y_1_3")))
Select(delaiElement).select_by_index(indexDelai)
print('---------------------TESTING DELAI SELECTED--------------------'+str(indexDelai))
for indexVerification in range(0, len(optionsVerification)):
driver.execute_script("document.getElementsByName('input_var_ZAKXXXXD_1_2')[0].style.display = 'block';")
verificationElement = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.NAME, "input_var_ZAKA834Y_1_3")))
Select(verificationElement).select_by_index(indexVerification)
print('---------------------TESTING VERIFICATION SELECTED--------------------'+str(indexVerification))
driver.execute_script("document.getElementsByName('input_var_PAKA840_1_1')[0].style.display = 'block';")
driver.execute_script("document.getElementsByName('input_var_PAKA840_2_1')[0].style.display = 'block';")
driver.execute_script("document.getElementsByName('input_var_ZAKA834Y_1_3')[0].style.display = 'block';")
driver.execute_script("document.getElementsByName('input_var_ZAKXXXXD_1_2')[0].style.display = 'block';")
price_update=driver.find_element_by_id('pr_basispreis')
with open('goodJobV2.csv', 'a') as csv_file:
writer = csv.writer(csv_file)
writer.writerow([optionsPaper[indexPaper], optionsTirage[indexTirage],optionsDelai[indexDelai],optionsVerification[indexVerification],price_update])
print('---------------------CLOSING FOR AND PAPER--------------------')
print('---------------------CONGRATULATIONS!!!!!--------------------')
driver.quit()
Hello Friends,
I am new to Python and Selenium, for a freelance project I need to create a web scraper that would automate the manual task of fetching the information from a website. I have added the code here. It is supposed to open a product page, trace through all possible combinations from the dropdown options and fetch each combination and the price of that product combination. And finally write it to an excel file. A product may have thousands of variations. Each time an option is selected on the dropdown, the price is updated. I have tried to used the WAIT and EXPECTED CONDITION in Selenium, however, I continue to get the exception "Message: stale element reference: element is not attached to the page document"
I have spent enough time on this, and count on the Stack overflow community to help me with a solution!
Thanks in advance!
