Merge pull request #1525 from seleniumbase/the-new-headless2-mode

"--headless2" to activate Chromium's new headless mode

Author: mdmintz

README.md

@@ -480,7 +480,8 @@ The code above will leave your browser window open in case there's a failure. (i
 --extension-dir=DIR  # (Load a Chrome Extension directory, comma-separated.)
 --pls=PLS  # (Set pageLoadStrategy on Chrome: "normal", "eager", or "none".)
 --headless  # (Run tests in headless mode. The default arg on Linux OS.)
---headed  # (Run tests in headed/GUI mode on Linux OS.)
+--headless2  # (Use the new headless mode, which supports extensions.)
+--headed  # (Run tests in headed/GUI mode on Linux OS, where not default.)
 --xvfb  # (Run tests using the Xvfb virtual display server on Linux OS.)
 --locale=LOCALE_CODE  # (Set the Language Locale Code for the web browser.)
 --interval=SECONDS  # (The autoplay interval for presentations & tour steps)

examples/raw_parameter_script.py

@@ -33,6 +33,7 @@
     sb.browser = "chrome"
     sb.is_behave = False
     sb.headless = False
+    sb.headless2 = False
     sb.headed = False
     sb.xvfb = False
     sb.start_page = None

help_docs/customizing_test_runs.md

@@ -134,7 +134,8 @@ pytest my_first_test.py --settings-file=custom_settings.py
 --extension-dir=DIR  # (Load a Chrome Extension directory, comma-separated.)
 --pls=PLS  # (Set pageLoadStrategy on Chrome: "normal", "eager", or "none".)
 --headless  # (Run tests in headless mode. The default arg on Linux OS.)
---headed  # (Run tests in headed/GUI mode on Linux OS.)
+--headless2  # (Use the new headless mode, which supports extensions.)
+--headed  # (Run tests in headed/GUI mode on Linux OS, where not default.)
 --xvfb  # (Run tests using the Xvfb virtual display server on Linux OS.)
 --locale=LOCALE_CODE  # (Set the Language Locale Code for the web browser.)
 --interval=SECONDS  # (The autoplay interval for presentations & tour steps)

mkdocs_build/requirements.txt

@@ -14,7 +14,7 @@ Jinja2==3.1.2
 click==8.1.3
 zipp==3.8.1
 ghp-import==2.1.0
-readme-renderer==37.1
+readme-renderer==37.2
 pymdown-extensions==9.5
 importlib-metadata==4.12.0
 bleach==5.0.1

requirements.txt

@@ -5,7 +5,7 @@ packaging>=20.9;python_version<"3.6"
 packaging>=21.3;python_version>="3.6"
 setuptools>=44.1.1;python_version<"3.6"
 setuptools>=59.6.0;python_version>="3.6" and python_version<"3.7"
-setuptools>=65.3.0;python_version>="3.7"
+setuptools>=65.4.0;python_version>="3.7"
 tomli>=1.2.3;python_version>="3.6" and python_version<"3.7"
 tomli>=2.0.1;python_version>="3.7"
 tqdm>=4.64.1
@@ -16,7 +16,7 @@ PyYAML>=6.0;python_version>="3.6"
 traitlets>=4.3.3;python_version<"3.7"
 traitlets>=5.4.0;python_version>="3.7"
 certifi>=2021.10.8;python_version<"3.6"
-certifi>=2022.9.14;python_version>="3.6"
+certifi>=2022.9.24;python_version>="3.6"
 filelock>=3.2.1;python_version<"3.6"
 filelock>=3.4.1;python_version>="3.6" and python_version<"3.7"
 filelock>=3.8.0;python_version>="3.7"
@@ -46,12 +46,12 @@ requests==2.27.1;python_version>="3.6" and python_version<"3.7"
 requests==2.28.1;python_version>="3.7"
 nose==1.3.7
 sniffio==1.3.0;python_version>="3.7"
-h11==0.13.0;python_version>="3.7"
+h11==0.14.0;python_version>="3.7"
 outcome==1.2.0;python_version>="3.7"
 trio==0.21.0;python_version>="3.7"
 trio-websocket==0.9.2;python_version>="3.7"
 websockets==10.3;python_version>="3.7"
-pyopenssl==22.0.0;python_version>="3.7"
+pyopenssl==22.1.0;python_version>="3.7"
 wsproto==1.2.0;python_version>="3.7"
 selenium==3.141.0;python_version<"3.7"
 selenium==4.4.3;python_version>="3.7"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "4.4.3"
+__version__ = "4.4.4"

seleniumbase/behave/behave_sb.py

@@ -43,7 +43,8 @@
 -D extension-dir=DIR  (Load a Chrome Extension directory, comma-separated.)
 -D pls=PLS  (Set pageLoadStrategy on Chrome: "normal", "eager", or "none".)
 -D headless  (Run tests in headless mode. The default arg on Linux OS.)
--D headed  (Run tests in headed/GUI mode on Linux OS.)
+-D headless2  (Use the new headless mode, which supports extensions.)
+-D headed  (Run tests in headed/GUI mode on Linux OS, where not default.)
 -D xvfb  (Run tests using the Xvfb virtual display server on Linux OS.)
 -D locale=LOCALE_CODE  (Set the Language Locale Code for the web browser.)
 -D pdb  (Activate Post Mortem Debug Mode if a test fails.)
@@ -131,6 +132,7 @@ def get_configured_sb(context):
     sb.browser = "chrome"
     sb.is_behave = True
     sb.headless = False
+    sb.headless2 = False
     sb.headless_active = False
     sb.headed = False
     sb.xvfb = False
@@ -262,6 +264,10 @@ def get_configured_sb(context):
         if low_key == "headless":
             sb.headless = True
             continue
+        # Handle: -D headless2
+        if low_key == "headless2":
+            sb.headless2 = True
+            continue
         # Handle: -D headed / gui
         if low_key in ["headed", "gui"]:
             sb.headed = True
@@ -733,36 +739,41 @@ def get_configured_sb(context):
             "\nOnly ONE default browser is allowed!\n"
             "%s browsers were selected: %s" % (len(browsers), browsers)
         )
-    # Recorder Mode does not support headless browser runs.
-    # Chromium does not allow extensions in Headless Mode.
+    # Recorder Mode can still optimize scripts in "-D headless2" mode.
     if sb.recorder_ext and sb.headless:
-        raise Exception(
-            "\n\n  Recorder Mode does NOT support Headless Mode!"
-            '\n  (DO NOT combine "-D rec" with "-D headless"!)\n'
-        )
+        sb.headless = False
+        sb.headless2 = True
+    if sb.browser not in ["chrome", "edge"]:
+        sb.headless2 = False  # Only for Chromium browsers
     # Recorder Mode only supports Chromium browsers.
     if sb.recorder_ext and (sb.browser not in ["chrome", "edge"]):
         raise Exception(
             "\n\n  Recorder Mode ONLY supports Chrome and Edge!"
             '\n  (Your browser choice was: "%s")\n' % sb.browser
         )
+    # Recorder Mode can still optimize scripts in --headless2 mode.
+    if sb.recorder_mode and sb.headless:
+        sb.headless = False
+        sb.headless2 = True
     # The Xvfb virtual display server is for Linux OS Only.
     if sb.xvfb and "linux" not in sys.platform:
         sb.xvfb = False
     if (
         "linux" in sys.platform
         and not sb.headed
         and not sb.headless
+        and not sb.headless2
         and not sb.xvfb
     ):
         print(
             '(Linux uses "-D headless" by default. '
             'To override, use "-D headed" / "-D gui". '
             'For Xvfb mode instead, use "-D xvfb". '
-            'Or hide this info with "-D headless".)'
+            'Or hide this info with "-D headless",'
+            'or by calling the new "-D headless2".)'
         )
         sb.headless = True
-    if not sb.headless:
+    if not sb.headless and not sb.headless2:
         sb.headed = True
     if sb.servername != "localhost":
         # Using Selenium Grid

seleniumbase/core/browser_launcher.py

@@ -333,6 +333,7 @@ def _set_chrome_options(
     undetectable,
     no_sandbox,
     disable_gpu,
+    headless2,
     incognito,
     guest_mode,
     devtools,
@@ -505,7 +506,9 @@ def _set_chrome_options(
                 "--disable-blink-features=AutomationControlled"
             )
         chrome_options.add_experimental_option("useAutomationExtension", False)
-    if headless:
+    if headless2:
+        chrome_options.add_argument("--headless=chrome")
+    elif headless:
         chrome_options.add_argument("--headless")
     if (settings.DISABLE_CSP_ON_CHROME or disable_csp) and not headless:
         # Headless Chrome does not support extensions, which are required
@@ -865,6 +868,7 @@ def get_driver(
     undetectable=False,
     no_sandbox=False,
     disable_gpu=False,
+    headless2=False,
     incognito=False,
     guest_mode=False,
     devtools=False,
@@ -943,20 +947,43 @@ def get_driver(
             raise Exception('The proxy PAC URL must end with ".pac"!')
         if proxy_pac_url and proxy_user and proxy_pass:
             proxy_auth = True
+    if (
+        is_using_uc(undetectable, browser_name)
+        and "linux" not in PLATFORM
+        and headless
+    ):
+        headless = False
+        headless2 = True
     if (
         headless
-        and proxy_auth
+        and (
+            proxy_auth
+            or ad_block_on
+            or disable_csp
+            or recorder_ext
+            or extension_zip
+            or extension_dir
+        )
         and (
             browser_name == constants.Browser.GOOGLE_CHROME
             or browser_name == constants.Browser.EDGE
         )
     ):
         # Headless Chrome/Edge doesn't support extensions, which are
-        # required when using a proxy server that has authentication.
+        # required when using a proxy server that has authentication,
+        # or when using other SeleniumBase extensions (eg: Recorder).
         # Instead, base_case.py will use the SBVirtualDisplay when not
         # using Chrome's built-in headless mode. See link for details:
         # https://bugs.chromium.org/p/chromium/issues/detail?id=706008
         headless = False
+        if "linux" not in PLATFORM:
+            # Use the new headless mode on Chrome if not using Linux:
+            # bugs.chromium.org/p/chromium/issues/detail?id=706008#c36
+            # Although Linux is technically supported, there are a lot
+            # of old versions of Chrome on Linux server machines, and
+            # this mode requires a recent version of Chrome to work.
+            # Specify "--headless2" as a pytest arg to use on Linux.
+            headless2 = True
     if (
         browser_name == constants.Browser.GOOGLE_CHROME
         and user_data_dir
@@ -990,6 +1017,7 @@ def get_driver(
             undetectable,
             no_sandbox,
             disable_gpu,
+            headless2,
             incognito,
             guest_mode,
             devtools,
@@ -1033,6 +1061,7 @@ def get_driver(
             undetectable,
             no_sandbox,
             disable_gpu,
+            headless2,
             incognito,
             guest_mode,
             devtools,
@@ -1080,6 +1109,7 @@ def get_remote_driver(
     undetectable,
     no_sandbox,
     disable_gpu,
+    headless2,
     incognito,
     guest_mode,
     devtools,
@@ -1177,6 +1207,7 @@ def get_remote_driver(
             undetectable,
             no_sandbox,
             disable_gpu,
+            headless2,
             incognito,
             guest_mode,
             devtools,
@@ -1402,6 +1433,7 @@ def get_remote_driver(
             undetectable,
             no_sandbox,
             disable_gpu,
+            headless2,
             incognito,
             guest_mode,
             devtools,
@@ -1594,6 +1626,7 @@ def get_local_driver(
     undetectable,
     no_sandbox,
     disable_gpu,
+    headless2,
     incognito,
     guest_mode,
     devtools,
@@ -1893,7 +1926,9 @@ def get_local_driver(
             edge_options.add_argument("--disable-sync")
         if guest_mode:
             edge_options.add_argument("--guest")
-        if headless:
+        if headless2:
+            edge_options.add_argument("--headless=chrome")
+        elif headless:
             edge_options.add_argument("--headless")
         if mobile_emulator:
             emulator_settings = {}
@@ -2177,6 +2212,7 @@ def get_local_driver(
                 undetectable,
                 no_sandbox,
                 disable_gpu,
+                headless2,
                 incognito,
                 guest_mode,
                 devtools,
@@ -2237,6 +2273,7 @@ def get_local_driver(
                 undetectable,
                 no_sandbox,
                 disable_gpu,
+                headless2,
                 incognito,
                 guest_mode,
                 devtools,
@@ -2551,6 +2588,7 @@ def get_local_driver(
                         undetectable,
                         no_sandbox,
                         disable_gpu,
+                        headless2,
                         incognito,
                         guest_mode,
                         devtools,