[EXPERIMENT] chore: run e2e with chromedriver

.github/workflows/e2e.yml

@@ -39,14 +39,15 @@ jobs:
         run: 'exit 1'
       - run: 'exit 0'
   e2e:
-    name: ${{ matrix.this_chunk }}/${{ matrix.total_chunks }} ${{ matrix.os }}-${{ matrix.head }}
+    name: ${{ matrix.this_chunk }}/${{ matrix.total_chunks }} ${{ matrix.kind }}-${{ matrix.os }}-${{ matrix.head }}
     strategy:
       fail-fast: false
       matrix:
         # TODO(#876): Add Windows CI.
         # TODO(#2154): Use macos-latest once Python setup supports mac arm64.
         os: [ubuntu-latest, macos-13]
         head: [headful, 'new-headless', 'old-headless']
+        kind: [chromedriver, mapper]
         total_chunks: [4]
         this_chunk: [0, 1, 2, 3]
         exclude:
@@ -89,6 +90,7 @@ jobs:
           --total-chunks=${{ matrix.total_chunks }}
         env:
           VERBOSE: ${{ github.event.inputs.verbose }}
+          CHROMEDRIVER: ${{ matrix.kind == 'chromedriver' }}
       - name: Run E2E tests
         if: matrix.os != 'ubuntu-latest' || matrix.head != 'headful'
         timeout-minutes: 20
@@ -99,9 +101,10 @@ jobs:
           --total-chunks=${{ matrix.total_chunks }}
         env:
           VERBOSE: ${{ github.event.inputs.verbose }}
+          CHROMEDRIVER: ${{ matrix.kind == 'chromedriver' }}
       - name: Upload artifacts
         if: always()
         uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
         with:
-          name: ${{ matrix.os }}-${{ matrix.head }}-${{ matrix.this_chunk }}-artifacts
+          name: ${{ matrix.kind }}-${{ matrix.os }}-${{ matrix.head }}-${{ matrix.this_chunk }}-artifacts
           path: logs

tests/browsing_context/test_close.py

@@ -13,13 +13,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import pytest
+import websockets
 from anys import ANY_DICT, ANY_STR
 from test_helpers import (AnyExtending, execute_command, get_tree, goto_url,
                           read_JSON_message, send_JSON_command, subscribe)
 
 
 @pytest.mark.asyncio
-async def test_browsingContext_close(websocket, context_id):
+async def test_browsingContext_close_last_command(websocket, context_id):
     await subscribe(websocket, ["browsingContext.contextDestroyed"])
 
     command_id = await send_JSON_command(websocket, {
@@ -46,10 +47,51 @@ async def test_browsingContext_close(websocket, context_id):
     resp = await read_JSON_message(websocket)
     assert resp == {"type": "success", "id": command_id, "result": {}}
 
+    try:
+        result = await get_tree(websocket)
+        assert result['contexts'] == []
+    except websockets.exceptions.ConnectionClosedError:
+        # Chromedriver closes the connection after the last context (not
+        # counting the mapper tab) is closed. NodeJS runner does not.
+        pass
+
+
+@pytest.mark.asyncio
+async def test_browsingContext_close_not_last_command(websocket, context_id,
+                                                      another_context_id):
+    await subscribe(websocket, ["browsingContext.contextDestroyed"])
+
+    command_id = await send_JSON_command(
+        websocket, {
+            "method": "browsingContext.close",
+            "params": {
+                "context": another_context_id
+            }
+        })
+
+    # Assert "browsingContext.contextCreated" event emitted.
+    resp = await read_JSON_message(websocket)
+    assert resp == {
+        'type': 'event',
+        "method": "browsingContext.contextDestroyed",
+        "params": {
+            "context": another_context_id,
+            "parent": None,
+            "url": "about:blank",
+            "children": None,
+            "userContext": "default"
+        }
+    }
+
+    resp = await read_JSON_message(websocket)
+    assert resp == {"type": "success", "id": command_id, "result": {}}
+
     result = await get_tree(websocket)
 
-    # Assert context is closed.
-    assert result == {'contexts': []}
+    # Assert only one context is left.
+    assert result == AnyExtending({'contexts': [{
+        "context": context_id,
+    }]})
 
 
 @pytest.mark.asyncio

tests/script/test_get_realms.py

@@ -18,10 +18,12 @@
 
 
 @pytest.mark.asyncio
-async def test_scriptGetRealms(websocket, context_id):
+async def test_scriptGetRealms(websocket, context_id, another_context_id):
     result = await execute_command(websocket, {
         "method": "script.getRealms",
-        "params": {}
+        "params": {
+            "context": context_id
+        }
     })
 
     assert {
@@ -56,7 +58,9 @@ async def test_scriptGetRealms(websocket, context_id):
 
     result = await execute_command(websocket, {
         "method": "script.getRealms",
-        "params": {}
+        "params": {
+            "context": context_id
+        }
     })
 
     assert ["realms"] == list(result.keys())
@@ -92,5 +96,5 @@ def realm_key(x):
         "params": {}
     })
 
-    # Assert no more realms existed.
-    assert {"realms": []} == result
+    # Assert only realm form the another tab is present.
+    assert len(result["realms"]) == 1

tests/session/test_protocol_basics.py

@@ -15,36 +15,14 @@
 
 import pytest
 from anys import ANY_STR
-from test_helpers import (execute_command, json, read_JSON_message,
-                          send_JSON_command)
+from test_helpers import (AnyExtending, execute_command, json,
+                          read_JSON_message, send_JSON_command)
 
 # Tests for "handle an incoming message" error handling, when the message
 # can't be decoded as known command.
 # https://w3c.github.io/webdriver-bidi/#handle-an-incoming-message
 
 
-@pytest.mark.asyncio
-async def test_binary(websocket):
-    # session.status is used in this test, but any simple command without side
-    # effects would work. It is first sent as text, which should work, and then
-    # sent again as binary, which should get an error response instead.
-    command = {"id": 1, "method": "session.status", "params": {}}
-
-    text_msg = json.dumps(command)
-    await websocket.send(text_msg)
-    resp = await read_JSON_message(websocket)
-    assert resp['id'] == 1
-
-    binary_msg = b'text_msg'
-    await websocket.send(binary_msg)
-    resp = await read_JSON_message(websocket)
-    assert resp == {
-        "type": "error",
-        "error": "invalid argument",
-        "message": "not supported type (binary)"
-    }
-
-
 @pytest.mark.asyncio
 async def test_invalid_json(websocket):
     message = 'this is not json'
@@ -66,7 +44,7 @@ async def test_empty_object(websocket):
     assert resp == {
         "type": "error",
         "error": "invalid argument",
-        "message": "Expected unsigned integer but got undefined"
+        "message": ANY_STR
     }
 
 
@@ -80,35 +58,58 @@ async def test_session_status(websocket):
     }
     await send_JSON_command(websocket, command)
     resp = await read_JSON_message(websocket)
-    assert resp == {
+    assert resp == AnyExtending({
         "id": 5,
         "type": "success",
         "result": {
             "ready": False,
             "message": "already connected"
         }
-    }
+    })
+
+
+@pytest.mark.asyncio
+async def test_channel_non_empty_static_command(websocket):
+    command_id = await send_JSON_command(websocket, {
+        "method": "session.status",
+        "params": {},
+        "channel": "SOME_CHANNEL"
+    })
+    resp = await read_JSON_message(websocket)
+
+    if "build" in resp["result"]:
+        # Heuristic to detect chromedriver.
+        pytest.xfail(reason="TODO: http://b/343683918")
+
+    assert resp == AnyExtending({
+        "id": command_id,
+        "channel": "SOME_CHANNEL",
+        "type": "success",
+        "result": {
+            "ready": False,
+            "message": "already connected"
+        }
+    })
 
 
 @pytest.mark.asyncio
-async def test_channel_non_empty(websocket):
-    await send_JSON_command(
+async def test_channel_non_empty_not_static_command(websocket):
+    command_id = await send_JSON_command(
         websocket, {
-            "id": 6000,
-            "method": "session.status",
+            "id": 2,
+            "method": "browsingContext.getTree",
             "params": {},
             "channel": "SOME_CHANNEL"
         })
     resp = await read_JSON_message(websocket)
-    assert resp == {
-        "id": 6000,
+    assert resp == AnyExtending({
+        "id": command_id,
         "channel": "SOME_CHANNEL",
         "type": "success",
         "result": {
-            "ready": False,
-            "message": "already connected"
+            "contexts": [{}]
         }
-    }
+    })
 
 
 @pytest.mark.asyncio
@@ -120,14 +121,14 @@ async def test_channel_empty(websocket):
         "channel": ""
     })
     resp = await read_JSON_message(websocket)
-    assert resp == {
+    assert resp == AnyExtending({
         "id": 7000,
         "type": "success",
         "result": {
             "ready": False,
             "message": "already connected"
         }
-    }
+    })
 
 
 @pytest.mark.asyncio

tests/session/test_subscription.py

@@ -412,11 +412,15 @@ async def test_subscribeToMultipleChannels_eventsReceivedInProperOrder(
 
     # Empty string channel is considered as no channel provided.
     resp = await read_JSON_message(websocket)
-    assert {
-        "type": "event",
-        "method": "log.entryAdded",
-        "params": ANY_DICT
-    } == resp
+    if "channel" in resp:
+        # Chromedriver adds channel even if it is an empty string.
+        pytest.xfail("TODO: http://b/343698990")
+    else:
+        assert {
+            "type": "event",
+            "method": "log.entryAdded",
+            "params": ANY_DICT
+        } == resp
 
     resp = await read_JSON_message(websocket)
     assert {