From 711e8217f59c3acce7102ff669acd235266221b5 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:38:18 +0100 Subject: [PATCH 01/17] :bug: fix passing a list instead of tuple for multipart file upload --- src/niquests/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/niquests/models.py b/src/niquests/models.py index e1633d496a..1f24e6e3f9 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -885,15 +885,15 @@ def _encode_files( ft: str | None = None fh: HeadersType | None = None - if isinstance(fdescriptor, tuple): + if isinstance(fdescriptor, (tuple, list)): # mypy and tuple length cmp not supported # https://github.com/python/mypy/issues/1178 if len(fdescriptor) == 2: - fn, fp = fdescriptor # type: ignore[misc] + fn, fp = tuple(fdescriptor) # type: ignore[misc,assignment] elif len(fdescriptor) == 3: - fn, fp, ft = fdescriptor # type: ignore[misc] + fn, fp, ft = tuple(fdescriptor) # type: ignore[misc,assignment] else: - fn, fp, ft, fh = fdescriptor # type: ignore[misc] + fn, fp, ft, fh = tuple(fdescriptor) # type: ignore[misc,assignment] else: if isinstance(fdescriptor, (str, bytes, bytearray)): fn = fkey From 75350a81d531e915d5959e4952b9afd0316b4140 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:40:15 +0100 Subject: [PATCH 02/17] :sparkle: ensure built-in support for SSE via urllib3-future 2.12.900+ --- .pre-commit-config.yaml | 2 +- docs/user/quickstart.rst | 81 ++++++++++++++++++++++++++++++++++++++++ pyproject.toml | 4 +- src/niquests/_async.py | 6 ++- src/niquests/adapters.py | 6 ++- src/niquests/models.py | 6 +++ src/niquests/sessions.py | 2 +- src/niquests/utils.py | 8 ++-- tests/test_sse.py | 48 ++++++++++++++++++++++++ 9 files changed, 151 insertions(+), 12 deletions(-) create mode 100644 tests/test_sse.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b22eda2971..d05f5c1f17 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -28,4 +28,4 @@ repos: - id: mypy args: [--check-untyped-defs] exclude: 'tests/|noxfile.py' - additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.11.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] + additional_dependencies: ['charset_normalizer', 'urllib3.future>=2.12.900', 'wassima>=1.0.1', 'idna', 'kiss_headers'] diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index deb82503bc..f8ca836baa 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1265,6 +1265,87 @@ See:: .. note:: The given example are really basic ones. You may adjust at will the settings and algorithm to match your requisites. +Server Side Event (SSE) +----------------------- + +.. note:: Available since version 3.11.2+ + +Server side event or widely known with its acronym SSE is a extremely popular method to stream continuously event +from the server to the client in real time. + +Before this built-in feature, most way to leverage this were to induce a bit of hacks into your http client. + +Starting example +~~~~~~~~~~~~~~~~ + +Thanks to urllib3-future native SSE extension, we can effortlessly manage a stream of event. +Here is a really basic example of how to proceed:: + + import niquests + + if __name__ == "__main__": + + r = niquests.post("sse://httpbingo.org/sse") + + print(r) # output: + + while r.extension.closed is False: + print(r.extension.next_payload()) # ServerSentEvent(event='ping', data='{"id":0,"timestamp":1732857000473}') + +We purposely set the scheme to ``sse://`` to indicate our intent to consume a SSE endpoint. + +.. note:: ``sse://`` is using ``https://`` under the hood. To avoid using an encrypted connection, use ``psse://`` instead. + +You will notice that the program is similar to our ``WebSocket`` implementation. Excepted that the ``next_payload()`` +method returns by default a ``ServerSentEvent`` object. + +Extracting raw event +~~~~~~~~~~~~~~~~~~~~ + +In the case where your server weren't compliant to the defined web standard for SSE (e.g. add custom field/line style) +you can extract a ``str`` instead of a ``ServerSentEvent`` object by passing ``raw=True`` into our ``next_payload()`` +method. + +As such:: + + while r.extension.closed is False: + print(r.extension.next_payload(raw=True)) # "event: ping\ndata: {"id":9,"timestamp":1732857471733}\n\n" + +Interrupt the stream +~~~~~~~~~~~~~~~~~~~~ + +A server may send event forever. And to avoid the awkward situation where your client receive unsolicited data +you should at all time close the SSE extension to notify the remote peer about your intent to stop. + +For example, the following test server send events until you say to stop: ``sse://sse.dev/test`` + +See how to stop cleanly the flow of events:: + + import niquests + + if __name__ == "__main__": + + r = niquests.post("sse://sse.dev/test") + + events = [] + + while r.extension.closed is False: + event = r.extension.next_payload() + + if event is None: # the remote peer closed it himself + break + + events.append(event) # add the event to list + + if len(events) >= 10: # close ourselves SSE stream & notify remote peer. + r.extension.close() + +Notes +~~~~~ + +SSE can be reached from HTTP/1, HTTP/2 or HTTP/3 at will. Niquests makes this very easy. +Moreover every features like proxies, happy-eyeballs, hooks, etc.. can be used as you always did. + ----------------------- Ready for more? Check out the :ref:`advanced ` section. diff --git a/pyproject.toml b/pyproject.toml index 1bc5b544e3..2c116e0f59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ description = "Niquests is a simple, yet elegant, HTTP library. It is a drop-in readme = "README.md" license-files = { paths = ["LICENSE"] } license = "Apache-2.0" -keywords = ["requests", "http2", "http3", "QUIC", "http", "https", "http client", "http/1.1", "ocsp", "revocation", "tls", "multiplexed", "dns-over-quic", "doq", "dns-over-tls", "dot", "dns-over-https", "doh", "dnssec"] +keywords = ["requests", "http2", "http3", "QUIC", "http", "https", "http client", "http/1.1", "ocsp", "revocation", "tls", "multiplexed", "dns-over-quic", "doq", "dns-over-tls", "dot", "dns-over-https", "doh", "dnssec", "websocket", "sse"] authors = [ {name = "Kenneth Reitz", email = "me@kennethreitz.org"} ] @@ -42,7 +42,7 @@ dynamic = ["version"] dependencies = [ "charset_normalizer>=2,<4", "idna>=2.5,<4", - "urllib3.future>=2.11.900,<3", + "urllib3.future>=2.12.900,<3", "wassima>=1.0.1,<2", "kiss_headers>=2,<4", ] diff --git a/src/niquests/_async.py b/src/niquests/_async.py index 9fb1ca4728..e082f84912 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -579,10 +579,12 @@ async def _redirect_method_ref(x, y): if not stream: if isinstance(r, AsyncResponse): - await r.content + if r.extension is None: + await r.content _swap_context(r) else: - r.content + if r.extension is None: + r.content return r diff --git a/src/niquests/adapters.py b/src/niquests/adapters.py index f2a7562bff..9201b165f4 100644 --- a/src/niquests/adapters.py +++ b/src/niquests/adapters.py @@ -1201,7 +1201,8 @@ def on_post_connection(conn_info: ConnectionInfo) -> None: extract_cookies_to_jar(session_cookies, sub_resp.request, sub_resp.raw) if not stream: - response.content + if response.extension is None: + response.content del self._promises[response_promise.uid] @@ -2304,7 +2305,8 @@ async def on_post_connection(conn_info: ConnectionInfo) -> None: extract_cookies_to_jar(session_cookies, sub_resp.request, sub_resp.raw) if not stream: - await response.content + if response.extension is None: + await response.content _swap_context(response) del self._promises[response_promise.uid] diff --git a/src/niquests/models.py b/src/niquests/models.py index 1f24e6e3f9..53febcb59e 100644 --- a/src/niquests/models.py +++ b/src/niquests/models.py @@ -57,10 +57,12 @@ from urllib3.contrib.webextensions._async import ( AsyncWebSocketExtensionFromHTTP, AsyncRawExtensionFromHTTP, + AsyncServerSideEventExtensionFromHTTP, ) from urllib3.contrib.webextensions import ( WebSocketExtensionFromHTTP, RawExtensionFromHTTP, + ServerSideEventExtensionFromHTTP, ) else: from urllib3_future import ( # type: ignore[assignment] @@ -82,10 +84,12 @@ from urllib3_future.contrib.webextensions._async import ( # type: ignore[assignment] AsyncWebSocketExtensionFromHTTP, AsyncRawExtensionFromHTTP, + AsyncServerSideEventExtensionFromHTTP, ) from urllib3_future.contrib.webextensions import ( # type: ignore[assignment] WebSocketExtensionFromHTTP, RawExtensionFromHTTP, + ServerSideEventExtensionFromHTTP, ) from ._typing import ( @@ -1058,8 +1062,10 @@ def extension( ) -> ( WebSocketExtensionFromHTTP | RawExtensionFromHTTP + | ServerSideEventExtensionFromHTTP | AsyncWebSocketExtensionFromHTTP | AsyncRawExtensionFromHTTP + | AsyncServerSideEventExtensionFromHTTP | None ): """Access the I/O after an Upgraded connection. E.g. for a WebSocket handler. diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index eafccfbef2..c55382b806 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -1343,7 +1343,7 @@ def on_early_response(early_response) -> None: except StopIteration: pass - if not stream: + if not stream and r.extension is None: r.content return r diff --git a/src/niquests/utils.py b/src/niquests/utils.py index d95863fce6..3aabc2db3c 100644 --- a/src/niquests/utils.py +++ b/src/niquests/utils.py @@ -1283,9 +1283,9 @@ def wrap_extension_for_http( ) class _WrappedExtensionFromHTTP(extension): # type: ignore[valid-type,misc] - def next_payload(self) -> str | bytes | None: + def next_payload(self, *args, **kwargs) -> str | bytes | None: try: - return super().next_payload() + return super().next_payload(*args, **kwargs) except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: @@ -1378,9 +1378,9 @@ def async_wrap_extension_for_http( ) class _AsyncWrappedExtensionFromHTTP(extension): # type: ignore[valid-type,misc] - async def next_payload(self) -> str | bytes | None: + async def next_payload(self, *args, **kwargs) -> str | bytes | None: try: - return await super().next_payload() + return await super().next_payload(*args, **kwargs) except ProtocolError as e: raise ChunkedEncodingError(e) except DecodeError as e: diff --git a/tests/test_sse.py b/tests/test_sse.py new file mode 100644 index 0000000000..5842fe6dfa --- /dev/null +++ b/tests/test_sse.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +import pytest + +from niquests import Session, AsyncSession + + + +@pytest.mark.usefixtures("requires_wan") +class TestLiveSSE: + def test_sync_sse_basic_example(self) -> None: + with Session() as s: + resp = s.get("sse://httpbingo.org/sse") + + assert resp.status_code == 200 + assert resp.extension is not None + assert resp.extension.closed is False + + events = [] + + while resp.extension.closed is False: + events.append( + resp.extension.next_payload() + ) + + assert resp.extension.closed is True + assert len(events) > 0 + assert events[-1] is None + + @pytest.mark.asyncio + async def test_async_sse_basic_example(self) -> None: + async with AsyncSession() as s: + resp = await s.get("sse://httpbingo.org/sse") + + assert resp.status_code == 200 + assert resp.extension is not None + assert resp.extension.closed is False + + events = [] + + while resp.extension.closed is False: + events.append( + await resp.extension.next_payload() + ) + + assert resp.extension.closed is True + assert len(events) > 0 + assert events[-1] is None From 57962b027edc297ba5fe6d1fa6bdf53cdbcaaa3b Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:41:07 +0100 Subject: [PATCH 03/17] :heavy_check_mark: add test case for ws read timeout (async&sync) --- tests/test_websocket.py | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/tests/test_websocket.py b/tests/test_websocket.py index 799548acff..736e50749d 100644 --- a/tests/test_websocket.py +++ b/tests/test_websocket.py @@ -2,7 +2,7 @@ import pytest -from niquests import Session, AsyncSession +from niquests import Session, AsyncSession, ReadTimeout try: import wsproto @@ -57,3 +57,42 @@ async def test_async_websocket_basic_example(self) -> None: await resp.extension.close() assert resp.extension.closed is True + + def test_sync_websocket_read_timeout(self) -> None: + with Session() as s: + resp = s.get("wss://echo.websocket.org", timeout=3) + + assert resp.status_code == 101 + assert resp.extension is not None + assert resp.extension.closed is False + + greeting_msg = resp.extension.next_payload() + + assert greeting_msg is not None + assert isinstance(greeting_msg, str) + + with pytest.raises(ReadTimeout): + resp.extension.next_payload() + + resp.extension.close() + assert resp.extension.closed is True + + @pytest.mark.asyncio + async def test_async_websocket_read_timeout(self) -> None: + async with AsyncSession() as s: + resp = await s.get("wss://echo.websocket.org", timeout=3) + + assert resp.status_code == 101 + assert resp.extension is not None + assert resp.extension.closed is False + + greeting_msg = await resp.extension.next_payload() + + assert greeting_msg is not None + assert isinstance(greeting_msg, str) + + with pytest.raises(ReadTimeout): + await resp.extension.next_payload() + + await resp.extension.close() + assert resp.extension.closed is True From 232f2bf845fd9c71e975dcdffaa5f08629cee3dd Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:41:32 +0100 Subject: [PATCH 04/17] :bookmark: bump version to 3.11.2 --- src/niquests/__version__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/niquests/__version__.py b/src/niquests/__version__.py index 3de8b4490a..2795da34bd 100644 --- a/src/niquests/__version__.py +++ b/src/niquests/__version__.py @@ -9,9 +9,9 @@ __url__: str = "https://niquests.readthedocs.io" __version__: str -__version__ = "3.11.1" +__version__ = "3.11.2" -__build__: int = 0x031101 +__build__: int = 0x031102 __author__: str = "Kenneth Reitz" __author_email__: str = "me@kennethreitz.org" __license__: str = "Apache-2.0" From 849fa50943c591b43be19210207b4625ad7aa419 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:41:44 +0100 Subject: [PATCH 05/17] :pencil: write changelog for 3.11.2 --- HISTORY.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 6a47c4313e..e30321549e 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,16 @@ Release History =============== +3.11.2 (2024-11-28) +------------------- + +**Fixed** +- SSE request block IO by default. Integrate better with urllib3-future new SSE web extension. +- Passing a list instead of tuple for multipart file upload ends in failure. + +**Changed** +- urllib3-future lower bound version is raised to 2.12.900 to ensure built-in support for SSE. + 3.11.1 (2024-11-22) ------------------- From 10a99e5f3dd172bb2d4ce401397b5a3d7cb72050 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:42:11 +0100 Subject: [PATCH 06/17] :pencil: update README for SSE --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 3d80f1c663..876a5b7355 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,8 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanced**” Python HTTP Client. Production Ready! ✔️ **Try before you switch:** [See Multiplexed in Action](https://replit.com/@ahmedtahri4/Python#main.py)
-📖 **See why you should switch:** [Read about 10 reasons why](https://medium.com/@ahmed.tahri/10-reasons-you-should-quit-your-http-client-98fd4c94bef3), and ["_Revived the promise made six years ago for Requests 3_"](https://medium.com/@ahmed.tahri/revived-the-promise-made-six-years-ago-for-requests-3-37b440e6a064) +📖 **See why you should switch:** [Read about 10 reasons why](https://medium.com/@ahmed.tahri/10-reasons-you-should-quit-your-http-client-98fd4c94bef3), and ["_Revived the promise made six years ago for Requests 3_"](https://medium.com/@ahmed.tahri/revived-the-promise-made-six-years-ago-for-requests-3-37b440e6a064)
+✨ **You were used to betamax, requests-mock, responses, ...?** [See how they still work! We got you covered.](https://niquests.readthedocs.io/en/latest/community/extensions.html)
👆 Look at the feature table comparison against requests, httpx and aiohttp! @@ -47,6 +48,7 @@ Niquests, is the “**Safest**, **Fastest[^10]**, **Easiest**, and **Most advanc | `WebSocket over HTTP/2 and HTTP/3` | ✅[^13] | ❌ | ❌ | ❌ | | `Automatic Ping for HTTP/2+` | ✅ | N/A | ❌ | N/A | | `Automatic Connection Upgrade / Downgrade` | ✅ | N/A | ❌ | N/A | +| `Server Side Event (SSE)` | ✅ | ❌ | ❌ | ❌ |
@@ -75,14 +77,12 @@ Did you give up on HTTP/2 due to performance concerns? Think again! Do you reali Multiplexing and response lazyness open up a wide range of possibilities! Want to learn more about the tests? scripts? reasoning? Take a deeper look at https://github.com/Ousret/niquests-stats - -⚠️ Do the responsible thing with this library and do not attempt DoS remote servers using its abilities.
```python >>> import niquests >>> s = niquests.Session(resolver="doh+google://", multiplexed=True) ->>> r = s.get('https://pie.dev/basic-auth/user/pass', auth=('user', 'pass')) +>>> r = s.get('https://one.one.one.one') >>> r >>> r.status_code @@ -111,7 +111,7 @@ import asyncio async def main() -> None: async with niquests.AsyncSession(resolver="doh+google://") as s: - r = await s.get('https://pie.dev/basic-auth/user/pass', auth=('user', 'pass'), stream=True) + r = await s.get('https://one.one.one.one', stream=True) print(r) # Output: payload = await r.json() print(payload) # Output: {'authenticated': True, ...} @@ -173,6 +173,7 @@ Niquests is ready for the demands of building scalable, robust and reliable HTTP - Trailers! - DNSSEC! - Async! +- SSE! Need something more? Create an issue, we listen. From b8284f3ee7d32119a7f14bebf47b7984ae347366 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:42:34 +0100 Subject: [PATCH 07/17] :art: reformat test_sse.py --- tests/test_sse.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/test_sse.py b/tests/test_sse.py index 5842fe6dfa..a601ebe13a 100644 --- a/tests/test_sse.py +++ b/tests/test_sse.py @@ -5,7 +5,6 @@ from niquests import Session, AsyncSession - @pytest.mark.usefixtures("requires_wan") class TestLiveSSE: def test_sync_sse_basic_example(self) -> None: @@ -19,9 +18,7 @@ def test_sync_sse_basic_example(self) -> None: events = [] while resp.extension.closed is False: - events.append( - resp.extension.next_payload() - ) + events.append(resp.extension.next_payload()) assert resp.extension.closed is True assert len(events) > 0 @@ -39,9 +36,7 @@ async def test_async_sse_basic_example(self) -> None: events = [] while resp.extension.closed is False: - events.append( - await resp.extension.next_payload() - ) + events.append(await resp.extension.next_payload()) assert resp.extension.closed is True assert len(events) > 0 From 19f73e70086a39041bba1edd30f591c2f8f37672 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:44:33 +0100 Subject: [PATCH 08/17] :pencil: update release date for 3.11.2 --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index e30321549e..a5a4d87a5d 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,7 +1,7 @@ Release History =============== -3.11.2 (2024-11-28) +3.11.2 (2024-11-29) ------------------- **Fixed** From f5db098ee810729d36220cb827e8cef6fafb2acc Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 06:52:22 +0100 Subject: [PATCH 09/17] :wrench: fix CI --- requirements-dev.txt | 3 ++- tests/test_ocsp.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index bf529793aa..10e964451e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,5 +5,6 @@ pytest-httpbin>=2,<3 pytest-asyncio>=0.21.1,<1.0 httpbin==0.10.2 trustme -cryptography<40.0.0; python_version <= '3.7' and platform_python_implementation == 'PyPy' +cryptography<40.0.0; python_version <= '3.7' +cryptography<44; python_version > '3.8' werkzeug>=2.2.3,<=3.0.2 diff --git a/tests/test_ocsp.py b/tests/test_ocsp.py index 1ea84b3aa2..1511b56ce8 100644 --- a/tests/test_ocsp.py +++ b/tests/test_ocsp.py @@ -22,9 +22,9 @@ class TestOnlineCertificateRevocationProtocol: @pytest.mark.parametrize( "revoked_peer_url", [ - # "https://revoked.badssl.com/", + "https://revoked.badssl.com/", # "https://revoked-rsa-ev.ssl.com/", - "https://revoked-ecc-dv.ssl.com/", + # "https://revoked-ecc-dv.ssl.com/", ], ) def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @@ -46,9 +46,9 @@ def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @pytest.mark.parametrize( "revoked_peer_url", [ - # "https://revoked.badssl.com/", + "https://revoked.badssl.com/", # "https://revoked-rsa-ev.ssl.com/", - "https://revoked-ecc-dv.ssl.com/", + # "https://revoked-ecc-dv.ssl.com/", ], ) async def test_async_revoked_certificate(self, revoked_peer_url: str) -> None: From 2d2af8ebd32983211f26d22b53b6d41f75510f1b Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:23:01 +0100 Subject: [PATCH 10/17] :wrench: switch to comodo test server for ocsp --- tests/test_ocsp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_ocsp.py b/tests/test_ocsp.py index 1511b56ce8..b3152d3bac 100644 --- a/tests/test_ocsp.py +++ b/tests/test_ocsp.py @@ -22,9 +22,10 @@ class TestOnlineCertificateRevocationProtocol: @pytest.mark.parametrize( "revoked_peer_url", [ - "https://revoked.badssl.com/", + # "https://revoked.badssl.com/", # "https://revoked-rsa-ev.ssl.com/", # "https://revoked-ecc-dv.ssl.com/", + "https://aaacertificateservices.comodoca.com:444/", ], ) def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @@ -46,9 +47,10 @@ def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @pytest.mark.parametrize( "revoked_peer_url", [ - "https://revoked.badssl.com/", + # "https://revoked.badssl.com/", # "https://revoked-rsa-ev.ssl.com/", # "https://revoked-ecc-dv.ssl.com/", + "https://aaacertificateservices.comodoca.com:444/", ], ) async def test_async_revoked_certificate(self, revoked_peer_url: str) -> None: From f1404f8eaab7e25df762f054c876b120be40091d Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:24:24 +0100 Subject: [PATCH 11/17] :wrench: add secondary ev sectigo revoked cert --- tests/test_ocsp.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_ocsp.py b/tests/test_ocsp.py index b3152d3bac..1f50632346 100644 --- a/tests/test_ocsp.py +++ b/tests/test_ocsp.py @@ -26,6 +26,7 @@ class TestOnlineCertificateRevocationProtocol: # "https://revoked-rsa-ev.ssl.com/", # "https://revoked-ecc-dv.ssl.com/", "https://aaacertificateservices.comodoca.com:444/", + "https://sectigopublicserverauthenticationrootr46-ev.sectigo.com:444/", ], ) def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @@ -51,6 +52,7 @@ def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: # "https://revoked-rsa-ev.ssl.com/", # "https://revoked-ecc-dv.ssl.com/", "https://aaacertificateservices.comodoca.com:444/", + "https://sectigopublicserverauthenticationrootr46-ev.sectigo.com:444/", ], ) async def test_async_revoked_certificate(self, revoked_peer_url: str) -> None: From 69bbcb879194d5cfa37a766810c1632fb3424151 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:29:49 +0100 Subject: [PATCH 12/17] :rewind: revert add secondary ev sectigo revoked cert --- tests/test_ocsp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_ocsp.py b/tests/test_ocsp.py index 1f50632346..b3152d3bac 100644 --- a/tests/test_ocsp.py +++ b/tests/test_ocsp.py @@ -26,7 +26,6 @@ class TestOnlineCertificateRevocationProtocol: # "https://revoked-rsa-ev.ssl.com/", # "https://revoked-ecc-dv.ssl.com/", "https://aaacertificateservices.comodoca.com:444/", - "https://sectigopublicserverauthenticationrootr46-ev.sectigo.com:444/", ], ) def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: @@ -52,7 +51,6 @@ def test_sync_revoked_certificate(self, revoked_peer_url: str) -> None: # "https://revoked-rsa-ev.ssl.com/", # "https://revoked-ecc-dv.ssl.com/", "https://aaacertificateservices.comodoca.com:444/", - "https://sectigopublicserverauthenticationrootr46-ev.sectigo.com:444/", ], ) async def test_async_revoked_certificate(self, revoked_peer_url: str) -> None: From 16f548c9528fab2617000338c69a94a904394b23 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:37:39 +0100 Subject: [PATCH 13/17] :wrench: review cryptography pin --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 10e964451e..831b26c81a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,6 +5,6 @@ pytest-httpbin>=2,<3 pytest-asyncio>=0.21.1,<1.0 httpbin==0.10.2 trustme -cryptography<40.0.0; python_version <= '3.7' +cryptography<40.0.0; python_version <= '3.8' cryptography<44; python_version > '3.8' werkzeug>=2.2.3,<=3.0.2 From 7078e74ce911900a6363d733e5ca008d4b1100f5 Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:57:57 +0100 Subject: [PATCH 14/17] :art: improve help script (add ws support + ssl full version) --- src/niquests/help.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/niquests/help.py b/src/niquests/help.py index cd5f0abc39..a262d2d823 100644 --- a/src/niquests/help.py +++ b/src/niquests/help.py @@ -4,7 +4,12 @@ import json import platform -import ssl + +try: + import ssl +except ImportError: + ssl = None # type: ignore + import sys import warnings from json import JSONDecodeError @@ -47,6 +52,11 @@ except ImportError: ocsp_verify = None # type: ignore +try: + import wsproto # type: ignore[import-not-found] +except ImportError: + wsproto = None # type: ignore + def _implementation(): """Return a dict with the Python implementation and version. @@ -108,10 +118,15 @@ def info(): "version": getattr(idna, "__version__", ""), } - system_ssl = ssl.OPENSSL_VERSION_NUMBER - system_ssl_info = { - "version": f"{system_ssl:x}" if system_ssl is not None else "N/A" - } + if ssl is not None: + system_ssl = ssl.OPENSSL_VERSION_NUMBER + + system_ssl_info = { + "version": f"{system_ssl:x}" if system_ssl is not None else "N/A", + "name": ssl.OPENSSL_VERSION, + } + else: + system_ssl_info = {"version": "N/A", "name": "N/A"} return { "platform": platform_info, @@ -139,6 +154,10 @@ def info(): "version": wassima.__version__, }, "ocsp": {"enabled": ocsp_verify is not None}, + "websocket": { + "enabled": wsproto is not None, + "wsproto": wsproto.__version__ if wsproto is not None else None, + }, } @@ -177,6 +196,7 @@ def check_update(package_name: str, actual_version: str) -> None: "charset-normalizer": charset_normalizer.__version__, "wassima": wassima.__version__, "idna": idna.__version__, + "wsproto": wsproto.__version__ if wsproto is not None else None, } From 3973f5e21cf23c48df7aa77962f5e234284ab40b Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:58:27 +0100 Subject: [PATCH 15/17] :pencil: Add hint when ws extra isn't installed and try to reach ws endpoint --- src/niquests/_async.py | 11 ++++++++++- src/niquests/sessions.py | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/niquests/_async.py b/src/niquests/_async.py index e082f84912..754901caa2 100644 --- a/src/niquests/_async.py +++ b/src/niquests/_async.py @@ -324,8 +324,17 @@ def get_adapter(self, url: str) -> AsyncBaseAdapter: # type: ignore[override] except ImportError: pass + # add a hint if wss:// fails when Niquests document its support. + # they probably forgot about the extra. + if url.startswith("ws://") or url.startswith("wss://"): + additional_hint = " Did you forget to install the extra for WebSocket? Run `pip install niquests[ws]` to fix this." + else: + additional_hint = "" + # Nothing matches :-/ - raise InvalidSchema(f"No connection adapters were found for {url!r}") + raise InvalidSchema( + f"No connection adapters were found for {url!r}{additional_hint}" + ) async def send( # type: ignore[override] self, request: PreparedRequest, **kwargs: typing.Any diff --git a/src/niquests/sessions.py b/src/niquests/sessions.py index c55382b806..a1bfed1bbe 100644 --- a/src/niquests/sessions.py +++ b/src/niquests/sessions.py @@ -1426,8 +1426,17 @@ def get_adapter(self, url: str) -> BaseAdapter: except ImportError: pass + # add a hint if wss:// fails when Niquests document its support. + # they probably forgot about the extra. + if url.startswith("ws://") or url.startswith("wss://"): + additional_hint = " Did you forget to install the extra for WebSocket? Run `pip install niquests[ws]` to fix this." + else: + additional_hint = "" + # Nothing matches :-/ - raise InvalidSchema(f"No connection adapters were found for {url!r}") + raise InvalidSchema( + f"No connection adapters were found for {url!r}{additional_hint}" + ) def close(self) -> None: """Closes all adapters and as such the session""" From c5990734ab147c8f1d6bb26a6713df3df7f8576a Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 07:58:36 +0100 Subject: [PATCH 16/17] :pencil: update changelog --- HISTORY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index a5a4d87a5d..ffbeedbe3c 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -10,6 +10,8 @@ Release History **Changed** - urllib3-future lower bound version is raised to 2.12.900 to ensure built-in support for SSE. +- help script now yield if websocket extra is present and actual version. +- exception raised when no adapter were found now include a hint when the intent was websocket and extra isn't installed. 3.11.1 (2024-11-22) ------------------- From 670e919cdd6f17c83c8dff8a017bf9337add8daf Mon Sep 17 00:00:00 2001 From: Ahmed TAHRI Date: Fri, 29 Nov 2024 08:12:37 +0100 Subject: [PATCH 17/17] :pencil: add details about ServerSentEvent object in docs --- docs/user/quickstart.rst | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/docs/user/quickstart.rst b/docs/user/quickstart.rst index f8ca836baa..72aaeb8f23 100644 --- a/docs/user/quickstart.rst +++ b/docs/user/quickstart.rst @@ -1311,6 +1311,8 @@ As such:: while r.extension.closed is False: print(r.extension.next_payload(raw=True)) # "event: ping\ndata: {"id":9,"timestamp":1732857471733}\n\n" +.. warning:: As with WebSocket, ``next_payload`` method may return None if the server terminate the stream. + Interrupt the stream ~~~~~~~~~~~~~~~~~~~~ @@ -1340,6 +1342,21 @@ See how to stop cleanly the flow of events:: if len(events) >= 10: # close ourselves SSE stream & notify remote peer. r.extension.close() +ServerSentEvent +~~~~~~~~~~~~~~~ + +.. note:: A ``ServerSentEvent`` object is returned by default with the ``next_payload()`` method. Or None if the server terminate the flow of events. + +It's a parsed SSE (single event). The object have nice shortcuts like: + +- ``payload.json()`` (any) to automatically unserialize passed json data. +- ``payload.id`` (str) +- ``payload.data`` (str) for the raw message payload +- ``payload.event`` (str) for the event type (e.g. message, ping, etc...) +- ``payload.retry`` (int) + +The full class source is located at https://github.com/jawah/urllib3.future/blob/3d7c5d9446880a8d473b9be4db0bcd419fb32dee/src/urllib3/contrib/webextensions/sse.py#L14 + Notes ~~~~~