I'm trying to do a webscraping. Until now I have the code to extract values from one page and change to the next page. But when I loop the process to do the same for all other pages it returns an error. Until now I have this code:
import time
import requests
import pandas
import pandas as pd
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import json
driver = webdriver.Chrome('C:\DRIVERS\chromedriver.exe')
driver.get('https://www.remax.pt/comprar?searchQueryState={%22regionName%22:%22%22,%22businessType%22:1,%22listingClass%22:1,%22page%22:1,%22sort%22:{%22fieldToSort%22:%22ContractDate%22,%22order%22:1},%22mapIsOpen%22:false}')
driver.maximize_window()
driver.implicitly_wait(5)
wait = WebDriverWait(driver, 10)
cookies = driver.find_element_by_id('rcc-decline-button')
cookies.click()
element_list = []
for j in range (1,2569):
try:
for i in range(1,40,2):
link = driver.find_element_by_xpath("(//div[@class='listing-search-searchdetails-component'])[{0}]".format(i))
link.click()
try:
detalhes = driver.find_element_by_id('details')
preço = driver.find_element_by_id('listing-price')
tipo = driver.find_element_by_id('listing-title')
freguesia = driver.find_element_by_xpath('//h5[@class="listing-address"]')
imoveis = [detalhes.text, preço.text, tipo.text, freguesia.text]
element_list.append(imoveis)
finally:
driver.back()
finally:
wait.until(EC.element_to_be_clickable((By.XPATH,"//a[@class='page-link'][.//span[.='Next']]"))).click()
All the values are scraped in the frist page but when it chages the page this error show up:
ERROR:
---------------------------------------------------------------------------
StaleElementReferenceException Traceback (most recent call last)
<ipython-input-7-052f5032275d> in <module>
12 for i in range(1,40,2):
13 link = driver.find_element_by_xpath("(//div[@class='listing-search-searchdetails-component'])[{0}]".format(i))
---> 14 link.click()
15 try:
16 detalhes = driver.find_element_by_id('details')
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in click(self)
78 def click(self):
79 """Clicks the element."""
---> 80 self._execute(Command.CLICK_ELEMENT)
81
82 def submit(self):
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webelement.py in _execute(self, command, params)
631 params = {}
632 params['id'] = self._id
--> 633 return self._parent.execute(command, params)
634
635 def find_element(self, by=By.ID, value=None):
~\anaconda3\lib\site-packages\selenium\webdriver\remote\webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
~\anaconda3\lib\site-packages\selenium\webdriver\remote\errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
(Session info: chrome=90.0.4430.72)
What element is this?
