Selenium / Accept cookie-window?

Question

i would like to automate the following site: https://atlas.immobilienscout24.de/

using this code:

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import os
from webdriver_manager.chrome import ChromeDriverManager

if __name__ == '__main__':    
  print(f"Checking Browser driver...")
  os.environ['WDM_LOG'] = '0' 
  options = Options()
  options.add_argument("start-maximized")
  options.add_experimental_option("prefs", {"profile.default_content_setting_values.notifications": 1})    
  options.add_experimental_option("excludeSwitches", ["enable-automation"])
  options.add_experimental_option('excludeSwitches', ['enable-logging'])
  options.add_experimental_option('useAutomationExtension', False)
  options.add_argument('--disable-blink-features=AutomationControlled')
  srv=Service(ChromeDriverManager().install())
  driver = webdriver.Chrome (service=srv, options=options)    
  waitWD = WebDriverWait (driver, 10)         
  
  link = f"https://atlas.immobilienscout24.de/" 
  driver.get (link)   
  driver.execute_script("arguments[0].click();", waitWD.until(EC.element_to_be_clickable((By.XPATH, '//button[@data-testid="uc-accept-all-button"]'))))          
  input("Press!")

When the site opens there allways appears this cookie-window and i am not able to press the accept button with the above code.

Instead i allways get this timeout-exception:

$ python test.py
Checking Browser driver...
Traceback (most recent call last):
  File "G:\DEV\Fiverr\TRY\mapesix\test.py", line 26, in <module>
    driver.execute_script("arguments[0].click();", waitWD.until(EC.element_to_be_clickable((By.XPATH, '//button[@data-testid="uc-accept-all-button"]'))))
  File "G:\DEV\.venv\selenium\lib\site-packages\selenium\webdriver\support\wait.py", line 90, in until
    raise TimeoutException(message, screen, stacktrace)
selenium.common.exceptions.TimeoutException: Message:
Stacktrace:
Backtrace:
        (No symbol) [0x00DB37D3]
        (No symbol) [0x00D48B81]
        (No symbol) [0x00C4B36D]
        (No symbol) [0x00C7D382]
        (No symbol) [0x00C7D4BB]
        (No symbol) [0x00CB3302]
        (No symbol) [0x00C9B464]
        (No symbol) [0x00CB1215]
        (No symbol) [0x00C9B216]
        (No symbol) [0x00C70D97]
        (No symbol) [0x00C7253D]
        GetHandleVerifier [0x0102ABF2+2510930]
        GetHandleVerifier [0x01058EC1+2700065]
        GetHandleVerifier [0x0105C86C+2714828]
        GetHandleVerifier [0x00E63480+645344]
        (No symbol) [0x00D50FD2]
        (No symbol) [0x00D56C68]
        (No symbol) [0x00D56D4B]
        (No symbol) [0x00D60D6B]
        BaseThreadInitThunk [0x766500F9+25]
        RtlGetAppContainerNamedObjectPath [0x77C27BBE+286]
        RtlGetAppContainerNamedObjectPath [0x77C27B8E+238]

How can i close this cookie-window?

Answer 1

The element Alle bestätigen is within #shadow-root (open)

---

Solution

To click on the element Alle akzeptieren you need to use shadow_root attribute and you can use the following locator strategies:

driver.get("https://atlas.immobilienscout24.de/")
time.sleep(7)
shadow_host = driver.find_element(By.CSS_SELECTOR, "div#usercentrics-root")
shadow_root = shadow_host.shadow_root
shadow_content = shadow_root.find_element(By.CSS_SELECTOR, "button[data-testid='uc-accept-all-button']")
shadow_content.click()
        

---

References

You can find a couple of relevant detailed discussions in:

• How to extract info within a #shadow-root (open) using Selenium Python?
• How to get past a cookie agreement page using Python and Selenium?

Answer 2

Finally managed how to fix the bug, you need a small time.sleep() to be able to click the button that is in a shadow_root element

from time import sleep
from typing import List
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import os
from webdriver_manager.chrome import ChromeDriverManager


if __name__ == '__main__':
    print(f"Checking Browser driver...")
    os.environ['WDM_LOG'] = '0'
    options = Options()
    options.add_argument("start-maximized")
    options.add_experimental_option(
        "prefs", {"profile.default_content_setting_values.notifications": 1})
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('excludeSwitches', ['enable-logging'])
    options.add_experimental_option('useAutomationExtension', False)
    options.add_argument('--disable-blink-features=AutomationControlled')
    srv = Service(ChromeDriverManager().install())
    driver = webdriver.Chrome(service=srv, options=options)
    waitWD = WebDriverWait(driver, 10)

    link = f"https://atlas.immobilienscout24.de/"
    driver.get(link)
    # accept cookie

    # Find the shadow host element
    shadow_host = driver.find_element(By.ID, "usercentrics-root")

    # Execute JavaScript to get the shadow root
    shadow_host = driver.find_element(By.ID, "usercentrics-root")

    # Execute JavaScript to get the shadow root
    shadow_root = driver.execute_script(
        'return arguments[0].shadowRoot', shadow_host)

    # Select the button element inside the shadow root using its data-testid attribute
    sleep(5)
    accept_all_button = driver.execute_script(
        'return arguments[0].querySelector(\'[data-testid="uc-accept-all-button"]\')', shadow_root
    )
    # Click the accept all button
    accept_all_button.click()
    input("accepted")

Tested twice and works in here, hope works there too!