GOAL
- Use Selenium in a JS continuously loading webpage via python web crawling task from an ec2 aws ubuntu 20.04 LTS instance
MAIN CODE PART
CHROME_PATH = '/usr/bin/chromium-browser'
CHROMEDRIVER_PATH = '/usr/bin/chromedriver'
WINDOW_SIZE = '1200, 800'
chrome_options = Options()
chrome_options.add_argument('headless') # chrome runs without a GUI window - as server doesn't have a gui
chrome_options.add_argument('window-size=%s' % WINDOW_SIZE)
#chrome_options.add_argument('ignore-ssl-errors')
chrome_options.add_argument('hide-scrollbars')
chrome_options.binary_location = CHROME_PATH
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(executable_path=CHROME_PATH,
options=chrome_options)
That I have tried to use afterwards
driver = webdriver.Chrome(
executable_path=CHROMEDRIVER_PATH,
chrome_options=chrome_options,
)
Warning message generated that is on for 1 min
<ipython-input-10-d3f251fa1d7a>:1: DeprecationWarning: use options instead of chrome_options
driver = webdriver.Chrome(
Than after 1 min error message
<ipython-input-8-d3f251fa1d7a>:1: DeprecationWarning: use options instead of chrome_options
driver = webdriver.Chrome(
---------------------------------------------------------------------------
WebDriverException Traceback (most recent call last)
<ipython-input-8-d3f251fa1d7a> in <module>
----> 1 driver = webdriver.Chrome(
2 executable_path=CHROMEDRIVER_PATH,
3 chrome_options=chrome_options,
4 )
5
/usr/local/lib/python3.8/dist-packages/selenium/webdriver/chrome/webdriver.py in __init__(self, executable_path, port, options, service_args, desired_capabilities, service_log_path, chrome_options, keep_alive)
74
75 try:
---> 76 RemoteWebDriver.__init__(
77 self,
78 command_executor=ChromeRemoteConnection(
/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/webdriver.py in __init__(self, command_executor, desired_capabilities, browser_profile, proxy, keep_alive, file_detector, options)
155 warnings.warn("Please use FirefoxOptions to set browser profile",
156 DeprecationWarning, stacklevel=2)
--> 157 self.start_session(capabilities, browser_profile)
158 self._switch_to = SwitchTo(self)
159 self._mobile = Mobile(self)
/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/webdriver.py in start_session(self, capabilities, browser_profile)
250 parameters = {"capabilities": w3c_caps,
251 "desiredCapabilities": capabilities}
--> 252 response = self.execute(Command.NEW_SESSION, parameters)
253 if 'sessionId' not in response:
254 response = response['value']
/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/webdriver.py in execute(self, driver_command, params)
319 response = self.command_executor.execute(driver_command, params)
320 if response:
--> 321 self.error_handler.check_response(response)
322 response['value'] = self._unwrap_value(
323 response.get('value', None))
/usr/local/lib/python3.8/dist-packages/selenium/webdriver/remote/errorhandler.py in check_response(self, response)
240 alert_text = value['alert'].get('text')
241 raise exception_class(message, screen, stacktrace, alert_text)
--> 242 raise exception_class(message, screen, stacktrace)
243
244 def _value_or_default(self, obj, key, default):
WebDriverException: Message: unknown error: DevToolsActivePort file doesn't exist
Solutions I have tried
pip install webdriver-manager
- DeprecationWarning: use options instead of chrome_options driver = webdriver.Chrome(chrome_options=options)
- Error message: "'chromedriver' executable needs to be available in the path"
- RECOMENDATION CODE
# selenium 4
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service
from webdriver_manager.chrome import ChromeDriverManager
options = webdriver.ChromeOptions()
options.add_argument("start-maximized")
driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH,service=Service(ChromeDriverManager().install()),options=options)
driver.get('https://www.google.com/')
ERORR
[WDM] - ====== WebDriver manager ======
2022-07-13 12:01:46,665 INFO ====== WebDriver manager ======
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
<ipython-input-15-14e7ef31432c> in <module>
8 options.add_argument("start-maximized")
9
---> 10 driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH,service=Service(ChromeDriverManager().install()),options=options)
11 driver.get('https://www.google.com/' class="ansi-blue-fg">)
~/.local/lib/python3.8/site-packages/webdriver_manager/chrome.py in install(self)
36
37 def install(self) -> str:
---> 38 driver_path = self._get_driver_path(self.driver)
39 os.chmod(driver_path, 0o755)
40 return driver_path
~/.local/lib/python3.8/site-packages/webdriver_manager/core/manager.py in _get_driver_path(self, driver)
27
28 def _get_driver_path(self, driver):
---> 29 binary_path = self.driver_cache.find_driver(driver)
30 if binary_path:
31 return binary_path
~/.local/lib/python3.8/site-packages/webdriver_manager/core/driver_cache.py in find_driver(self, driver)
93 os_type = driver.get_os_type()
94 driver_name = driver.get_name()
---> 95 driver_version = driver.get_version()
96 browser_version = driver.browser_version
97
~/.local/lib/python3.8/site-packages/webdriver_manager/core/driver.py in get_version(self)
41 def get_version(self):
42 self._version = (
---> 43 self.get_latest_release_version()
44 if self._version == "latest"
45 else self._version
~/.local/lib/python3.8/site-packages/webdriver_manager/drivers/chrome.py in get_latest_release_version(self)
35
36 def get_latest_release_version(self):
---> 37 self.browser_version = get_browser_version_from_os(self.chrome_type)
38 log(f"Get LATEST {self._name} version for {self.browser_version} {self.chrome_type}")
39 latest_release_url = (
~/.local/lib/python3.8/site-packages/webdriver_manager/core/utils.py in get_browser_version_from_os(browser_type)
150 return get_browser_version(browser_type, metadata)
151
--> 152 cmd_mapping = {
153 ChromeType.BRAVE: {
154 OSType.LINUX: linux_browser_apps_to_cmd(
KeyError: 'google-chrome'
