Why does this selenium script work for non-headless, but when I switch this to headless, it doesn't work?

Question

This script is supposed to go into booking.com and grab the first price of the waldorf astoria Las Vegas hotel. It completely works for when I'm opening a browser, but doesn't work after switching to headless mode.

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
import translateXMLfile
from flask import Flask, jsonify
from selenium.webdriver.chrome.options import Options

class BookingAutomation:
    WINDOW_SIZE = "1200,900"
    HEADLESS = True
    def __init__(self, driver):
        self.driver = driver

    def setup_driver(self):
        
        opts = Options()  # Use webdriver.ChromeOptions()

        if self.HEADLESS:
            opts.add_argument('--headless') #runs a headless search
            
        opts.add_argument("--window-size=%s" % self.WINDOW_SIZE)
        
        # self.driver =webdriver.Chrome()
        self.driver = webdriver.Chrome(opts)

        self.driver.maximize_window()
        self.driver.get('https://www.booking.com') #go to the booking.com url
        time.sleep(2)
        
        close_popup = self.driver.find_element(By.CSS_SELECTOR, "button[aria-label='Dismiss sign-in info.']") #close the popup that pops when you first enter the website
        close_popup.click()
        time.sleep(2)
        

    def select_dates(self):
        
        search = self.driver.find_element(By.ID,':re:') #find the search bar
        search.send_keys(translateXMLfile.name) # enter the waldorf astoria
        
        time.sleep(3)
        

        date = self.driver.find_element(By.CSS_SELECTOR, 'button[data-testid="date-display-field-end"]')
        date.click() #find the calendar and click it

        next_button = self.driver.find_element(By.CSS_SELECTOR,
                                               "button[class='a83ed08757 c21c56c305 f38b6daa18 d691166b09 f671049264 deab83296e f4552b6561 dc72a8413c f073249358']") #go into the next month
        next_button.click() #click the next button

        inday = translateXMLfile.iday #insert the before and after days
        oday = translateXMLfile.oday

        formatted_inday = f"{inday:02d}"
        formatted_oday = f"{oday:02d}"

        date_element_xpath = f"//span[contains(@aria-label, '{formatted_inday} October 2023')]"
        date_element_xpatho = f"//span[contains(@aria-label, '{formatted_oday} October 2023')]"

        checkin_date = self.driver.find_element(By.XPATH, date_element_xpath)
        checkout_date = self.driver.find_element(By.XPATH, date_element_xpatho)

        checkin_date.click()
        checkout_date.click()
        
        *************search.send_keys(Keys.ENTER) #click enter after entering the search and the date
        
        
       Inside these stars are where I believe my issue lies
        
    def search_hotels(self):
        self.driver.save_screenshot('2nd test.png') #this displays the image, somehow deleting what I wrote and the dates I've put in after I click enter*********************************************

        first_result = self.driver.find_element(By.CSS_SELECTOR, "div[class='c82435a4b8 a178069f51 a6ae3c2b40 a18aeea94d d794b7a0f7 f53e278e95 da89aeb942 c12ee2f811'] div[class='d20f4628d0'] div[class='b978843432'] div[class='aca0ade214 aaf30230d9 cd2e7d62b0 f996d8c258'] div div[class='aca0ade214 c9835feea9 c2931f4182 d79e71457a d14b211b4f'] div[class='aaee4e7cd3 e7a57abb1e'] div[class='aca0ade214 a5f1aae5b2 cd2e7d62b0'] div div[class='f6431b446c a23c043802']") ****** this fails to grab for some reason, but only in headless mode
        
        first_result.click()
        
        time.sleep(2)
        
    def check_prices(self):
        handles = []
        handles = self.driver.window_handles
        if len(handles)>1:
            self.driver.switch_to.window(handles[1])

        time.sleep(10)

        
       
        
        first_price = self.driver.find_element(By.XPATH, "//tr[contains(@class,'js-rt-block-row e2e-hprt-table-row hprt-table-cheapest-block hprt-table-cheapest-block-fix js-hprt-table-cheapest-block')]//div//div//span[@class='prco-valign-middle-helper'][normalize-space()='$2,177']")  # Replace with your XPath
        print(first_price.text)
        time.sleep(5)
        
    def close_driver(self):
        if self.driver:
            self.driver.quit()
if __name__ == "__main__":
    driver = webdriver.Chrome()
    booking_automation = BookingAutomation(driver)
    booking_automation.setup_driver()
    # booking_automation.check_height()
    booking_automation.select_dates()
    booking_automation.search_hotels()
    # booking_automation.check_prices()
    
    booking_automation.close_driver()

I have tried putting the date before the search, then clicking the enter button, I tried pressing enter on the keyboard, I tried waiting before clicking the button as well as the keyboard. If there is anyone that has the slightest clue how to solve this I would really appreciate it. I've been banging my head on this for hours now.

score 0 · Accepted Answer · answered Aug 28 '23 at 15:26

That's because you're using Chrome's old headless mode, which works differently from regular Chrome. Details here: https://stackoverflow.com/a/73840130/7058266

If you want headless Chrome to work the same as regular headed Chrome, then change this:

opts.add_argument("--headless")

To this:

opts.add_argument("--headless=new")

Why does this selenium script work for non-headless, but when I switch this to headless, it doesn't work?

1 Answers1