I am trying to close browser even if the script is not exited normally. I have tried few methods which I found on internet but failed to do so. First was to use try/except and finally. I implemented it but if does not works if the script execution is forcefully stopped. I have reproduced the code I have. It is not closing the browser on force exit if using chrome. Firefox is closing without any problem. The code is divided in two files (scraper.py, initialize.py)
scraper.py
try:
from .initialize import Init
except:
from initialize import Init
import time
URL = "https://web.whatsapp.com/"
def main(driver):
# do parsing and other stuff
driver.get(URL)
time.sleep(15)
def get_driver(browser, headless):
# calling start drivers method of init class from initialize file
return Init(method='profile',write_directory='output/',selenium_wire=True,undetected_chromedriver=True,old_profile_using=False).start_driver(browser=browser, url=URL, headless=headless, refresh=True)
try:
driver = get_driver("firefox" , False)
main(driver)
# get_driver returns driver.
except Exception as e:
print(f"Error: {e}")
finally:
try:
driver.quit()
except NameError:
pass
initialize.py
from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.chrome.service import Service
import shutil, os, sys, getpass, browser_cookie3, json, traceback
from time import sleep
from webdriver_manager.firefox import GeckoDriverManager
from webdriver_manager import utils
"""def drivers():
driver_installation = ChromeDriverManager().install()
service = Service(driver_installation)
driver = webdriver.Chrome(service=service)
return driver
class Init:
def __init__(self, browser):
self.browser = browser
def get_driver(self): #for sake of simplicity this is the get_driver function
driver = drivers()
return driver"""
class Init:
basename = os.path.basename(sys.argv[0]).split(".")[0]
homedir = os.path.join("C:\\ProgramData", basename, getpass.getuser())
#WRITE_DIRECTORY = 'output_files'
#WRITE_DIRECTORY = f'{homedir}\output_files/'
WRITE_DIRECTORY = os.path.join(homedir,'output_files')
def __init__(self, method, write_directory=WRITE_DIRECTORY, selenium_wire=False,
undetected_chromedriver=False, old_profile_using=False):
self.method = method
self.write_directory = write_directory
self.selenium_wire = selenium_wire
self.undetected_chromedriver = undetected_chromedriver
self.old_profile_using = old_profile_using
if not any((self.selenium_wire, self.undetected_chromedriver)):
raise Exception("Specify at least one driver")
if method not in ['profile', 'cookies']:
raise Exception(f"Unexpected method {method=}")
@staticmethod
def find_path_to_browser_profile(browser):
"""Getting path to firefox or chrome default profile"""
if sys.platform == 'win32':
browser_path = {
'chrome': os.path.join(
os.environ.get('LOCALAPPDATA'), 'Google', 'Chrome', 'User Data'),
"firefox": os.path.join(
os.environ.get('APPDATA'), 'Mozilla', 'Firefox')
}
else:
browser_path = {
'chrome': os.path.expanduser('~/.config/google-chrome'),
"firefox": os.path.expanduser('~/.mozilla/firefox')
}
path_to_profiles = browser_path[browser]
print(f'{path_to_profiles=}')
if os.path.exists(path_to_profiles):
if browser == 'firefox':
return browser_cookie3.Firefox.get_default_profile(path_to_profiles)
if browser == 'chrome':
return path_to_profiles
else:
print(f"{browser} profile - not found")
def _remove_old_db(self, path_to_browser_profile, browser):
if browser == 'firefox':
path_to_browser_profile = os.path.join(path_to_browser_profile, 'storage', 'default')
folder_list = [i for i in
os.listdir(path_to_browser_profile) if 'to-do.live' in i]
if folder_list:
path_to_folder = os.path.join(path_to_browser_profile, folder_list[0], 'idb')
try:
shutil.rmtree(path_to_folder)
print('removed')
return
except:
sleep(2)
pass
elif browser == 'chrome':
path_to_browser_profile = os.path.join(path_to_browser_profile, 'Default', 'IndexedDB')
else:
raise Exception('browser not supported')
print(path_to_browser_profile)
folder_list = [i for i in
os.listdir(path_to_browser_profile) if 'to-do.live' in i]
if folder_list:
print(folder_list[0])
path_to_folder = os.path.join(path_to_browser_profile, folder_list[0])
shutil.rmtree(path_to_folder)
print('removed')
def init_driver(self, browser, path_to_profile=None, headless=False, remove_old_session=False):
"""Initialize webdriver"""
print(f"Initialize webdriver for {browser}")
if self.selenium_wire and self.undetected_chromedriver:
from seleniumwire.undetected_chromedriver.v2 import Chrome, ChromeOptions # working
elif self.selenium_wire:
from seleniumwire.webdriver import Chrome, ChromeOptions # not working
else:
from undetected_chromedriver import Chrome, ChromeOptions
if self.method == 'profile':
path_to_profile = self.copy_profile(browser)
if remove_old_session:
self._remove_old_db(path_to_profile, browser)
if 'firefox' in browser:
wdm_path = os.path.join(os.path.expanduser("~"), ".wdm", "drivers.json")
print(wdm_path)
with open(wdm_path, "r") as f:
driver_logs = json.load(f)
gecko_drivers = {}
g_count = 0
for drivers, val in driver_logs.items():
if "geckodriver" in drivers:
g_count += 1
gecko_drivers[drivers] = val
firefox_options = webdriver.FirefoxOptions()
# firefox_options.add_argument('--no-sandbox')
if self.method == 'profile':
path_to_firefox_profile = path_to_profile
profile = webdriver.FirefoxProfile(path_to_firefox_profile)
profile.accept_untrusted_certs = True
profile.set_preference("dom.webdriver.enabled", False)
profile.set_preference('useAutomationExtension', False)
profile.set_preference("security.mixed_content.block_active_content", False)
profile.update_preferences()
firefox_options.set_preference("general.useragent.override", 'user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0')
#firefox_options.add_argument(
# 'user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0')
if headless:
firefox_options.add_argument("--headless")
firefox_options.add_argument('--ignore-certificate-errors')
firefox_options.add_argument("--width=1400")
firefox_options.add_argument("--height=1000")
try:
driver_installation = GeckoDriverManager().install()
except Exception as e:
c = 1
if "rate limit" in str(e):
for i in gecko_drivers.values():
if c==g_count:
driver_installation = i["binary_path"]
break
c +=1
service = Service(driver_installation)
if sys.platform == 'win32':
from subprocess import CREATE_NO_WINDOW
service.creationflags = CREATE_NO_WINDOW
driver = webdriver.Firefox(options=firefox_options, firefox_profile=profile,
service=service)
driver.implicitly_wait(10)
else:
firefox_options = webdriver.FirefoxOptions()
firefox_options.add_argument('--incognito')
firefox_options.add_argument("--disable-blink-features=AutomationControlled")
firefox_options.add_argument(
'user-agent=Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:89.0) Gecko/20100101 Firefox/89.0')
firefox_options.add_argument("--mute-audio")
if headless:
firefox_options.add_argument("--headless")
try:
driver_installation = GeckoDriverManager().install()
except Exception as e:
if "rate limit" in str(e):
c = 1
for i in gecko_drivers.values():
if c==g_count:
driver_installation = i["binary_path"]
c +=1
service = Service(driver_installation)
if sys.platform == 'win32':
from subprocess import CREATE_NO_WINDOW
service.creationflags = CREATE_NO_WINDOW
driver = webdriver.Firefox(options=firefox_options, executable_path=driver_installation,
service=service)
return driver
elif 'chrome' in browser:
chrome_options = ChromeOptions()
if self.method == 'profile':
chrome_options.add_argument('--no-first-run --no-service-autorun --password-store=basic')
chrome_options.add_argument("--disable-extensions")
# chrome_options.user_data_dir = path_to_profile
chrome_options.add_argument("--window-size=1400,1000")
chrome_options.add_argument(
'user-agent=Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36')
if sys.platform == 'linux':
chrome_options.binary_location = '/bin/google-chrome'
if headless:
chrome_options.add_argument("--headless")
# You cannot use default chromedriver for google services, because it was made by google.
browser_version = int(utils.get_browser_version_from_os('google-chrome').split('.')[0])
print(f"{browser_version=}")
driver = Chrome(options=chrome_options, version_main=browser_version, patcher_force_close=True,
user_data_dir=path_to_profile)
driver.implicitly_wait(10)
return driver
else:
chrome_options.add_argument('--incognito')
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument("--mute-audio")
chrome_options.add_argument("--window-size=1400,1000")
if headless:
chrome_options.add_argument("--headless")
chrome_options.add_argument(
'user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36')
driver_installation = ChromeDriverManager().install()
service = Service(driver_installation)
if sys.platform == 'win32':
from subprocess import CREATE_NO_WINDOW
service.creationflags = CREATE_NO_WINDOW
driver = webdriver.Chrome(driver_installation, options=chrome_options, service=service)
return driver
def copy_profile(self, browser):
"""Copy browsers profile to /home/{username}/browser_profiles/{browser} directory and returns path to it"""
path_to_profile = self.find_path_to_browser_profile(browser)
sys_path_to_copy = os.path.join(os.getcwd(), self.write_directory, browser)
print(f'copy_profile() initializer: sys_path_to_copy: {sys_path_to_copy}')
if browser == 'chrome':
if self.old_profile_using:
print('Using old profile')
return sys_path_to_copy
if not os.path.exists(sys_path_to_copy):
try:
shutil.copytree(path_to_profile, sys_path_to_copy,
symlinks=True, ignore_dangling_symlinks=True, dirs_exist_ok=True)
except:
pass
print(f'{browser} profile copied from {path_to_profile} to {sys_path_to_copy}')
if sys.platform == 'win32':
try:
shutil.copytree(os.path.join(sys_path_to_copy,'Default','Network'), os.path.join(sys_path_to_copy,'Default'),dirs_exist_ok=True)
except:
pass
return sys_path_to_copy
elif browser == 'firefox':
firefox_dir = path_to_profile.split('/')[-1]
if self.old_profile_using:
print('Using old profile')
return os.path.join(sys_path_to_copy, firefox_dir)
if not os.path.exists(sys_path_to_copy):
try:
shutil.copytree(path_to_profile, os.path.join(sys_path_to_copy, firefox_dir),
ignore_dangling_symlinks=True)
except:
pass
print(
f'{browser} profile copied from {path_to_profile} to {sys_path_to_copy}/{firefox_dir}')
return os.path.join(sys_path_to_copy, firefox_dir)
def start_driver(self, url, browser, headless, refresh=True, remove_old_session=False):
"""Prepering folders and init webdriver"""
self.create_folders(self.write_directory)
if self.method == "profile":
path_to_profile = f'{self.write_directory}{browser}'
if not self.old_profile_using:
self.remove_profile_folder(path_to_profile)
driver = self.init_driver(browser, headless=headless, remove_old_session=remove_old_session)
try:
sleep(3)
driver.get(url)
sleep(5)
if refresh and "telegram" not in url:
driver.refresh()
print(driver.current_url)
if driver.current_url == "about:blank":
driver.get(url)
sleep(5)
return driver
except:
# raise Exception
print(traceback.format_exc())
driver.close()
driver.quit()
def remove_profile_folder(self, path_to_profile_folder):
"""Removes browser profile folder"""
print("Removing old profile")
while os.path.exists(path_to_profile_folder):
try:
shutil.rmtree(path_to_profile_folder)
except Exception as ex:
print(ex)
sleep(2)
pass
def create_folders(self, folder_path):
if not os.path.exists(folder_path):
print(f'initializer() Creating folder for {folder_path}')
os.makedirs(folder_path) # Creating folder
return folder_path
Things I tried
try:
driver = get_driver(browser)
# get_driver returns driver.
main(
driver,
)
except:
print(traceback.format_exc())
finally:
try:
driver.quit()
except NameError:
pass
The other thing I tried was to use atexit
import atexit
def get_driver(browser): #for sake of simplicity this is the get_driver function
driver_installation = ChromeDriverManager().install()
service = Service(driver_installation)
driver = webdriver.Chrome(service=service)
return driver
@atexit.register()
def cleanup():
driver.close()
driver.quit()
try:
driver = get_driver(browser)
# get_driver returns driver.
main(
driver,
)
except:
print(traceback.format_exc())
finally:
try:
driver.quit()
except NameError:
pass
Note, I tries the above in my main code not like this. These also work fine in simple script but not in the code I have provided above. I do not know if I am using it correctly or not. The browser is closed on force exit if I am executing the script using VS Code debugger but it fails in simple execution. Thanks in advance for any help.
