how to scroll page by using selenium

Question

I saw a page on How can I scroll a web page using selenium webdriver in python? it can help me to scroll down to page end, but it has problem on some website. when you run this, the product list will no load, but manually scroll to the Viewallbutton(report1)睇更多(180) and tap it, then scroll you can get all the product.

how can i fix it? i want to load all the product.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.chrome.options import Options 

chrome_path = r'C:\Users\ecgoo\Desktop\program\Pycharm\chromedriver.exe'
driver = webdriver.Chrome(chrome_path, )

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--disable-popup-blocking")

driver.maximize_window()
driver.get("https://www.ztore.com/tc/category/group/snacks")
tabName = driver.find_element_by_link_text("零食新登場")
tabName.click()
time.sleep(1)

report1 = driver.find_element_by_xpath("/html/body/div[1]/div/div/div[3]/span/div/span")
print(report1)
report1.click()

time.sleep(10)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(300)

driver.close()

Answer 1

This should work:

def scroll_to_last():

    # get the height of the last scrollable position
    last_scroll_height = driver.execute_script('return document.body.scrollHeight')

    while True:

        # scroll down as long as the scrollbar is not in the last scrollable position/height
        driver.execute_script('window.scrollTo(0,document.body.scrollHeight)')
        
        time.sleep(5)

        # get the current height/position of the scrollbar
        new_height = driver.execute_script('return document.body.scrollHeight')

        # if the new height is equal to the last scrollable height, break the loop
        if new_height == last_scroll_height:
            break

        # else, take the last height as new height of the scrollbar
        last_scroll_height = new_height

Answer 2

I took the solution given in this link - you can go to this link and then read the logic behind this.

The code that I have for this which works - tested on my machine -

import time
from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.common.by import By
from webdriver_manager.chrome import ChromeDriverManager

svc=Service(ChromeDriverManager().install())
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--disable-popup-blocking")
driver = webdriver.Chrome(service=svc, options=chrome_options)
driver.maximize_window()
driver.get("https://www.ztore.com/tc/category/group/snacks")
time.sleep(10)  # Allow 10 seconds for the web page to open

driver.find_element(By.LINK_TEXT,"零食新登場").click()
time.sleep(5)
scroll_pause_time = 1 # You can set your own pause time. My laptop is a bit slow so I use 1 sec

screen_height = driver.execute_script("return window.screen.height;")   # get the screen height of the web
i = 1
count=0
driver.find_element(By.CSS_SELECTOR,"div.viewAllButton").click()

while True:
    # scroll one screen height each time
    driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))
    i += 1
    time.sleep(scroll_pause_time)
    # update scroll height each time after scrolled, as the scroll height can change after we scrolled the page
    scroll_height = driver.execute_script("return document.body.scrollHeight;")
    # Break the loop when the height we need to scroll to is larger than the total scroll height
    if (screen_height) * i > scroll_height:
        break


driver.quit()

The code has been done using the latest Beta release of Selenium 4, so you may see a couple of changes.