Python Selenium throws StaleElementReferenceException(The element reference is stale )

Question

I'm trying to scrape Stackoverflow using python selenium.

I get StaleElementReferenceException for few elements when I try to print the link(href) of each question.

Below is the exception which is seen

selenium.common.exceptions.StaleElementReferenceException: Message: The element reference is stale. Either the element is no longer attached to the DOM or the page has been refreshed.

I've tried both

WebDriverWait(driver, 3).until(EC.presence_of_element_located(
(By.XPATH, xpath)))

and

driver.implicitly_wait(30)

I have already checked few questions and couldn't solve this problem StaleElementReferenceException: Element is no longer attached to the DOM: Selenium

Code snippet

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By

driver = webdriver.Firefox()
driver.get("https://stackoverflow.com/questions")
try:
    WebDriverWait(driver, 3).until(EC.presence_of_element_located(
        (By.XPATH, '//div[@id="tabs"]/a[@class="youarehere"]')))
    print "Page is ready!"
except TimeoutException:
    print "Time exceeded"

noofquestionsperpage = driver.find_element_by_xpath(
    '//div[@class="page-sizer fr"]/a[@class="page-numbers current"]').text
requirednumberofposts = 30
numberofpagesclick = requirednumberofposts / int(noofquestionsperpage)

print numberofpagesclick
if numberofpagesclick > 1:
    for i in range(numberofpagesclick):
        for a in driver.find_elements_by_xpath('//*[@id="questions"]/div[@class="question-summary"]'):
            try:
                WebDriverWait(driver, 3).until(EC.presence_of_element_located(
                    (By.XPATH, './/div[@class="summary"]/h3/a')))
                print a.find_element_by_xpath('.//div[@class="summary"]/h3/a').get_attribute('href')
            except Exception, e:
                print str(e).strip()
        try:
            driver.find_element_by_xpath(
                '//div[@class="pager fl"]/a[6]/span[@class="page-numbers next"]').click()
        except:
            print "All questons loaded"

Update

Found an alternate way to iterate the questions in stackoverflow,posting the code.Now I'm generating xpath based on div id of each quesion instead of using questions webelement which was causing StaleElementReferenceException earlier.

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By


driver = webdriver.Firefox()
driver.get("https://stackoverflow.com/questions")
try:
    WebDriverWait(driver, 3).until(EC.presence_of_element_located(
        (By.XPATH, '//div[@id="tabs"]/a[@class="youarehere"]')))
    print "Page is ready!"
except TimeoutException:
    print "Time exceeded"

noofquestionsperpage = driver.find_element_by_xpath(
    '//div[@class="page-sizer fr"]/a[@class="page-numbers current"]').text
requirednumberofposts = 30
numberofpagesclick = requirednumberofposts / int(noofquestionsperpage)

if numberofpagesclick > 1:
    for i in range(numberofpagesclick):
        for j in range(int(noofquestionsperpage)):
            xpath = "//div[@id='questions']/div[" + \
                str(j + 1) + "]/div[2]/h3/a"
            print driver.find_element_by_xpath(xpath).get_attribute('href')
    try:
        driver.find_element_by_xpath(
            '//div[@class="pager fl"]/a[6]/span[@class="page-numbers next"]').click()
    except:
        print "All questons loaded"

Python Selenium throws StaleElementReferenceException(The element reference is stale )

0 Answers0