Merge pull request #2017 from seleniumbase/refactor-logging-and-driver-settings

Refactor logging and driver settings

Author: mdmintz

examples/raw_parameter_script.py

@@ -59,7 +59,6 @@
     sb.extension_zip = None
     sb.extension_dir = None
     sb.database_env = "test"
-    sb.log_path = "latest_logs"
     sb.archive_logs = False
     sb.disable_csp = False
     sb.disable_ws = False

mkdocs_build/requirements.txt

@@ -2,35 +2,25 @@
 # Minimum Python version: 3.8 (for generating docs only)
 
 regex>=2023.8.8
-pkginfo>=1.9.6
 PyYAML>=6.0.1
-readme-renderer>=41.0
 pymdown-extensions>=10.1
-importlib-metadata>=6.8.0
 pipdeptree>=2.13.0
-bleach>=6.0.0
-docutils>=0.20.1
 python-dateutil>=2.8.2
-tqdm>=4.66.1
-nltk>=3.8.1
-joblib>=1.3.2
-livereload==2.6.3
 Markdown==3.4.4
+markdown2==2.4.10
 MarkupSafe==2.1.3
 Jinja2==3.1.2
 click==8.1.7
 ghp-import==2.1.0
-lunr==0.7.0.post1
-tornado==6.3.3
 watchdog==3.0.0
 cairocffi==1.6.1
-cairosvg==2.7.1
-cssselect2==0.7.0
-tinycss2==1.2.1
-defusedxml==0.7.1
 pathspec==0.11.2
+Babel==2.12.1
+paginate==0.5.6
+pyquery==2.0.0
+readtime==3.0.0
 mkdocs==1.5.2
-mkdocs-material==9.1.21
+mkdocs-material==9.2.1
 mkdocs-exclude-search==0.6.5
 mkdocs-simple-hooks==0.1.5
 mkdocs-material-extensions==1.1.1

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.17.8"
+__version__ = "4.17.9"

seleniumbase/behave/behave_sb.py

@@ -168,7 +168,7 @@ def get_configured_sb(context):
     sb.driver_version = None
     sb.page_load_strategy = None
     sb.database_env = "test"
-    sb.log_path = "latest_logs" + os.sep
+    sb.log_path = constants.Logs.LATEST + os.sep
     sb.archive_logs = False
     sb.disable_js = False
     sb.disable_csp = False
@@ -940,7 +940,9 @@ def get_configured_sb(context):
     if sb_config.dash_title:
         constants.Dashboard.TITLE = sb_config.dash_title.replace("_", " ")
 
-    log_helper.log_folder_setup("latest_logs/", sb.archive_logs)
+    log_helper.log_folder_setup(
+        constants.Logs.LATEST + "/", sb.archive_logs
+    )
     download_helper.reset_downloads_folder()
     proxy_helper.remove_proxy_zip_if_present()
     return sb
@@ -1153,7 +1155,9 @@ def _perform_behave_unconfigure_():
                 pass
         sb_config.shared_driver = None
     if hasattr(sb_config, "archive_logs"):
-        log_helper.archive_logs_if_set("latest_logs/", sb_config.archive_logs)
+        log_helper.archive_logs_if_set(
+            constants.Logs.LATEST + "/", sb_config.archive_logs
+        )
     log_helper.clear_empty_logs()
     # Dashboard post-processing: Disable time-based refresh and stamp complete
     if not hasattr(sb_config, "dashboard") or not sb_config.dashboard:
@@ -1227,7 +1231,9 @@ def do_final_driver_cleanup_as_needed():
 
 
 def _perform_behave_terminal_summary_():
-    latest_logs_dir = os.path.join(os.getcwd(), "latest_logs" + os.sep)
+    latest_logs_dir = os.path.join(
+        os.getcwd(), constants.Logs.LATEST + os.sep
+    )
     dash_path = os.path.join(os.getcwd(), "dashboard.html")
     equals_len = len("Dashboard: ") + len(dash_path)
     try:

seleniumbase/core/browser_launcher.py

@@ -18,6 +18,7 @@
 from seleniumbase import drivers  # webdriver storage folder for SeleniumBase
 from seleniumbase import extensions  # browser extensions storage folder
 from seleniumbase.config import settings
+from seleniumbase.core import detect_b_ver
 from seleniumbase.core import download_helper
 from seleniumbase.core import proxy_helper
 from seleniumbase.fixtures import constants
@@ -190,8 +191,10 @@ def has_cf(text):
     if (
         "<title>Just a moment...</title>" in text
         or "<title>403 Forbidden</title>" in text
+        or "Permission Denied</title>" in text
         or 'id="challenge-error-text"' in text
         or 'action="/?__cf_chl_f_tk' in text
+        or 'src="chromedriver.js"' in text
         or 'id="challenge-form"' in text
         or "window._cf_chl_opt" in text
     ):
@@ -218,11 +221,11 @@ def uc_special_open_if_cf(driver, url, proxy_string=None):
             pass
         if special:
             with driver:
-                time.sleep(0.2)
+                time.sleep(0.18)
                 driver.execute_script('window.open("%s","_blank");' % url)
                 driver.close()
                 driver.switch_to.window(driver.window_handles[-1])
-                time.sleep(0.2)
+                time.sleep(0.02)
         else:
             driver.open(url)  # The original one
     else:
@@ -233,9 +236,9 @@ def uc_special_open_if_cf(driver, url, proxy_string=None):
 def uc_open(driver, url):
     if (url.startswith("http:") or url.startswith("https:")):
         with driver:
-            time.sleep(0.2)
+            time.sleep(0.18)
             driver.open(url)
-            time.sleep(0.2)
+            time.sleep(0.02)
     else:
         driver.open(url)  # The original one
     return None
@@ -244,11 +247,11 @@ def uc_open(driver, url):
 def uc_open_with_tab(driver, url):
     if (url.startswith("http:") or url.startswith("https:")):
         with driver:
-            time.sleep(0.2)
+            time.sleep(0.18)
             driver.execute_script('window.open("%s","_blank");' % url)
             driver.close()
             driver.switch_to.window(driver.window_handles[-1])
-            time.sleep(0.2)
+            time.sleep(0.02)
     else:
         driver.open(url)  # The original one
     return None
@@ -312,7 +315,7 @@ def get_valid_binary_names_for_browser(browser):
         else:
             raise Exception("Could not determine OS, or unsupported!")
     else:
-        raise Exception("Invalid combination for os browser binaries!")
+        raise Exception("Invalid combination for OS browser binaries!")
 
 
 def _repair_chromedriver(chrome_options, headless_options, mcv=None):
@@ -870,7 +873,12 @@ def _set_chrome_options(
                     )
                 except Exception:
                     pass
-            if len(chromium_arg_item) >= 3:
+            if "set-binary" in chromium_arg_item and not binary_location:
+                br_app = "google-chrome"
+                binary_loc = detect_b_ver.get_binary_location(br_app)
+                if os.path.exists(binary_loc):
+                    binary_location = binary_loc
+            elif len(chromium_arg_item) >= 3:
                 chrome_options.add_argument(chromium_arg_item)
     if devtools and not headless:
         chrome_options.add_argument("--auto-open-devtools-for-tabs")
@@ -1166,6 +1174,8 @@ def get_driver(
                 "that includes the driver filename at the end of it!"
                 "\n(Will use default settings...)\n" % binary_location
             )
+            # Example of a valid binary location path - MacOS:
+            # "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"
             binary_location = None
         else:
             binary_name = binary_location.split("/")[-1].split("\\")[-1]
@@ -2251,8 +2261,6 @@ def get_local_driver(
         use_version = "latest"
         major_edge_version = None
         try:
-            from seleniumbase.core import detect_b_ver
-
             if binary_location:
                 try:
                     major_edge_version = (
@@ -2480,8 +2488,6 @@ def get_local_driver(
         )
         edge_options.add_argument("--disable-browser-side-navigation")
         edge_options.add_argument("--disable-translate")
-        if binary_location:
-            edge_options.binary_location = binary_location
         if not enable_ws:
             edge_options.add_argument("--disable-web-security")
         edge_options.add_argument("--homepage=about:blank")
@@ -2585,8 +2591,15 @@ def get_local_driver(
                         chromium_arg_item = "-" + chromium_arg_item
                     else:
                         chromium_arg_item = "--" + chromium_arg_item
-                if len(chromium_arg_item) >= 3:
+                if "set-binary" in chromium_arg_item and not binary_location:
+                    br_app = "edge"
+                    binary_loc = detect_b_ver.get_binary_location(br_app)
+                    if os.path.exists(binary_loc):
+                        binary_location = binary_loc
+                elif len(chromium_arg_item) >= 3:
                     edge_options.add_argument(chromium_arg_item)
+        if binary_location:
+            edge_options.binary_location = binary_location
         if selenium4_or_newer:
             try:
                 service = EdgeService(
@@ -2884,8 +2897,6 @@ def get_local_driver(
             major_chrome_version = None
             if selenium4_or_newer:
                 try:
-                    from seleniumbase.core import detect_b_ver
-
                     if binary_location:
                         try:
                             major_chrome_version = (

seleniumbase/core/detect_b_ver.py

@@ -106,6 +106,68 @@ def windows_browser_apps_to_cmd(*apps):
     return '%s -NoProfile "%s"' % (powershell, script)
 
 
+def get_binary_location(browser_type):
+    cmd_mapping = {
+        ChromeType.GOOGLE: {
+            OSType.LINUX: linux_browser_apps_to_cmd(
+                "google-chrome-stable",
+                "google-chrome",
+                "chrome",
+                "chromium",
+                "chromium-browser",
+                "google-chrome-beta",
+                "google-chrome-dev",
+                "google-chrome-unstable",
+            ),
+            OSType.MAC: r"/Applications/Google Chrome.app"
+                        r"/Contents/MacOS/Google Chrome",
+            OSType.WIN: windows_browser_apps_to_cmd(
+                r'(Get-Item -Path "$env:PROGRAMFILES\Google\Chrome'
+                r'\Application\chrome.exe")',
+                r'(Get-Item -Path "$env:PROGRAMFILES (x86)\Google\Chrome'
+                r'\Application\chrome.exe")',
+                r'(Get-Item -Path "$env:LOCALAPPDATA\Google\Chrome'
+                r'\Application\chrome.exe")',
+            ),
+        },
+        ChromeType.MSEDGE: {
+            OSType.LINUX: linux_browser_apps_to_cmd(
+                "microsoft-edge-stable",
+                "microsoft-edge",
+                "microsoft-edge-beta",
+                "microsoft-edge-dev",
+            ),
+            OSType.MAC: r"/Applications/Microsoft Edge.app"
+                        r"/Contents/MacOS/Microsoft Edge",
+            OSType.WIN: windows_browser_apps_to_cmd(
+                # stable edge
+                r'(Get-Item -Path "$env:PROGRAMFILES\Microsoft\Edge'
+                r'\Application\msedge.exe")',
+                r'(Get-Item -Path "$env:PROGRAMFILES (x86)\Microsoft'
+                r'\Edge\Application\msedge.exe")',
+                # beta edge
+                r'(Get-Item -Path "$env:LOCALAPPDATA\Microsoft\Edge Beta'
+                r'\Application\msedge.exe")',
+                r'(Get-Item -Path "$env:PROGRAMFILES\Microsoft\Edge Beta'
+                r'\Application\msedge.exe")',
+                r'(Get-Item -Path "$env:PROGRAMFILES (x86)\Microsoft\Edge Beta'
+                r'\Application\msedge.exe")',
+                # dev edge
+                r'(Get-Item -Path "$env:LOCALAPPDATA\Microsoft\Edge Dev'
+                r'\Application\msedge.exe")',
+                r'(Get-Item -Path "$env:PROGRAMFILES\Microsoft\Edge Dev'
+                r'\Application\msedge.exe")',
+                r'(Get-Item -Path "$env:PROGRAMFILES (x86)\Microsoft\Edge Dev'
+                r'\Application\msedge.exe")',
+                # canary edge
+                r'(Get-Item -Path "$env:LOCALAPPDATA\Microsoft\Edge SxS'
+                r'\Application\msedge.exe")',
+            ),
+        },
+    }
+    return cmd_mapping[browser_type][os_name()]
+
+
 def get_browser_version_from_binary(binary_location):
     try:
         if binary_location.count(r"\ ") != binary_location.count(" "):
@@ -118,18 +180,19 @@ def get_browser_version_from_binary(binary_location):
         return None
 
 
-def get_browser_version_from_os(browser_type=None):
+def get_browser_version_from_os(browser_type):
     """Return installed browser version."""
     cmd_mapping = {
         ChromeType.GOOGLE: {
             OSType.LINUX: linux_browser_apps_to_cmd(
-                "google-chrome",
                 "google-chrome-stable",
+                "google-chrome",
                 "chrome",
                 "chromium",
+                "chromium-browser",
                 "google-chrome-beta",
                 "google-chrome-dev",
-                "chromium-browser",
+                "google-chrome-unstable",
             ),
             OSType.MAC: r"/Applications/Google\ Chrome.app"
                         r"/Contents/MacOS/Google\ Chrome --version",
@@ -149,8 +212,8 @@ def get_browser_version_from_os(browser_type=None):
         },
         ChromeType.MSEDGE: {
             OSType.LINUX: linux_browser_apps_to_cmd(
-                "microsoft-edge",
                 "microsoft-edge-stable",
+                "microsoft-edge",
                 "microsoft-edge-beta",
                 "microsoft-edge-dev",
             ),

seleniumbase/core/download_helper.py

@@ -11,8 +11,6 @@
 # The "downloads_folder" is cleaned out at the start of each pytest run,
 #     but there is an option to save existing files in "archived_files".
 DOWNLOADS_DIR = constants.Files.DOWNLOADS_FOLDER
-ARCHIVE_DIR = constants.Files.ARCHIVED_DOWNLOADS_FOLDER
-
 abs_path = os.path.abspath(".")
 downloads_path = os.path.join(abs_path, DOWNLOADS_DIR)
 
@@ -24,7 +22,19 @@ def get_downloads_folder():
 def reset_downloads_folder():
     """Clears the downloads folder.
     If settings.ARCHIVE_EXISTING_DOWNLOADS is set to True, archives it."""
-    archived_downloads_folder = os.path.join(os.getcwd(), ARCHIVE_DIR) + os.sep
+    downloads_dir = constants.Files.DOWNLOADS_FOLDER
+    archive_dir = constants.Files.ARCHIVED_DOWNLOADS_FOLDER
+    if downloads_dir.endswith("/"):
+        downloads_dir = downloads_dir[:-1]
+    if downloads_dir.startswith("/"):
+        downloads_dir = downloads_dir[1:]
+    if archive_dir.endswith("/"):
+        archive_dir = archive_dir[:-1]
+    if archive_dir.startswith("/"):
+        archive_dir = archive_dir[1:]
+    if len(downloads_dir) < 10 or len(archive_dir) < 10:
+        return  # Prevent accidental deletions if constants are renamed
+    archived_downloads_folder = os.path.join(os.getcwd(), archive_dir) + os.sep
     if os.path.exists(downloads_path) and not os.listdir(downloads_path) == []:
         reset_downloads_folder_assistant(archived_downloads_folder)
     if os.path.exists(downloads_path) and os.listdir(downloads_path) == []: