Scrolling using Selenium(4.10.0)

Question

I'm trying to scroll a table on a website I need to scroll because the information in the table is loaded dynamically.

If I try the send_keys(Keys.END) method, the element not interactable error is returned.

Or driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") - noting

Or ActionChains(driver) - nothing

Tried selecting different elements and classes and tags and XPATHs, CSS_SELECTORs

Python 3.8, Selenium(4.10.0)

from time import sleep
from selenium import webdriver
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as ES
from selenium.webdriver.support.ui import WebDriverWait as WDW
from selenium.webdriver.common.action_chains import ActionChains
from bs4 import BeautifulSoup as BS

self.URL_alnair = 'https://alnair.ae/app/view/1412/3386/apartment/apartments'
o = Options()
o.add_experimental_option('detach', True)
o.add_argument('--start-maximized')
chrome_driver_path = r"YOUR PATH"
driver = webdriver.Chrome(options=o,service=ChromeService(executable_path=chrome_driver_path))

def get_data():
    driver.get(self.URL_alnair)
    driver.set_page_load_timeout(2)
    
    element = driver.find_elements(By.CSS_SELECTOR, '._tr_18734_1')
    action.move_to_element(element[2])


def get_data():
    driver.get(self.URL_alnair)
    driver.set_page_load_timeout(2)
    
    driver.execute_script("window.scrollTo(0, 500)") # tried "document.body.scrollHeight", #"window.scrollBy(0, 200)", element.scrollTop = 500... etc


def get_data():
    driver.get(self.URL_alnair)
    driver.set_page_load_timeout(2)
    
    action = ActionChains(self.driver)

    element = self.driver.find_elements(By.CSS_SELECTOR, '._tr_18734_1')
    action.move_to_element(element[2]).perform()
    sleep(1)
    action.send_keys(Keys.ARROW_DOWN).perform() #

The above describes several code functions that I have tried in different ways.

Ajeet Verma · Accepted Answer · 2023-07-29T11:18:06.540

Here's how you may try:

from selenium.webdriver import Chrome
from selenium.webdriver.chrome.service import Service as ChromeService
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager


URL_alnair = 'https://alnair.ae/app/view/1412/3386/apartment/apartments'
o = Options()
o.add_experimental_option('detach', True)
o.add_argument('--start-maximized')

driver = Chrome(service=ChromeService(ChromeDriverManager().install()), options=o)

def get_data():
    driver.get(URL_alnair)
    driver.set_page_load_timeout(2)

    scroll_bar = driver.find_element(By.CSS_SELECTOR, 'div[class^="_scrollContainer_"]')
    driver.execute_script("arguments[0].scrollBy(0, arguments[0].scrollHeight);", scroll_bar)

get_data()

Things to note:

You first need to find/locate the scrollbar which is embedded in the HTML page.
The web-element <div class="_scrollContainer_1ah3s_14"> represents the scrollbar which can be located using Selenium WebDriver Finding Element by Partial Class Name:
```
scroll_bar = driver.find_element(By.CSS_SELECTOR, 'div[class^="_scrollContainer_"]')
```

Once we find the web element for the scrollbar, simply can scroll down to its height:

driver.execute_script("arguments[0].scrollBy(0, arguments[0].scrollHeight);", scroll_bar)

You may also try using webdriver-manager

pip install webdriver-manager

Purpose of webdriver manager

Scrolling using Selenium(4.10.0)

1 Answers1