diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst index 9a75f4986e1a3a9..a5a461abaa9a367 100644 --- a/Doc/library/urllib.request.rst +++ b/Doc/library/urllib.request.rst @@ -522,6 +522,31 @@ The following classes are provided: Process HTTP error responses. +.. class:: HTTPGzipHandler() + + Request and transparently decode gzip-compressed responses. + + This handler is not one of the handlers installed by :func:`build_opener` + by default; pass it explicitly to opt in:: + + opener = urllib.request.build_opener(urllib.request.HTTPGzipHandler) + with opener.open("http://www.example.com/") as response: + body = response.read() + + Unless the request already has an :mailheader:`Accept-Encoding` header, + the handler adds ``Accept-Encoding: gzip``. If the response is then sent + with ``Content-Encoding: gzip``, its body is decoded as it is read and the + :mailheader:`Content-Encoding` and :mailheader:`Content-Length` headers, + which describe the compressed body, are removed. If the request already + carries an :mailheader:`Accept-Encoding` header, the response is returned + unchanged, so a caller that asked for compressed data can decode it itself. + + Only ``gzip`` is supported. The :mod:`zlib` module is required; + constructing the handler without it raises :exc:`ImportError`. + + .. versionadded:: next + + .. _request-objects: Request Objects diff --git a/Doc/whatsnew/3.16.rst b/Doc/whatsnew/3.16.rst index ce001d07731bcc5..727ee2296700cee 100644 --- a/Doc/whatsnew/3.16.rst +++ b/Doc/whatsnew/3.16.rst @@ -901,6 +901,17 @@ unicodedata * The Unicode database has been updated to Unicode 18.0.0. +urllib.request +-------------- + +* Add :class:`~urllib.request.HTTPGzipHandler`, an opt-in handler that + requests gzip-compressed responses (by adding an ``Accept-Encoding: gzip`` + header) and transparently decodes them. It is not installed by + :func:`~urllib.request.build_opener` by default; add it explicitly to opt + in. + (Contributed by Julian Soreavis in :gh:`43521`.) + + uuid ---- diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 5bc8de9b82defbe..3c06b26570da2cd 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -2025,6 +2025,141 @@ def test_invalid_closed(self): self.assertTrue(conn.fakesock.closed, "Connection not closed") +class FakeGzipResponse: + # Minimal stand-in for the http.client.HTTPResponse that a response + # processor sees: headers plus a readable, closable body. + def __init__(self, data, headers, code=200): + self._file = io.BytesIO(data) + self.headers = headers + self.code = code + self.msg = "OK" + self.url = "http://example.com/" + self.closed = False + + def read(self, amt=-1): + return self._file.read(amt) + + def close(self): + self.closed = True + self._file.close() + + +@support.requires_zlib() +class HTTPGzipHandlerTests(unittest.TestCase): + + body = b"The quick brown fox jumps over the lazy dog.\n" * 200 + + def make_response(self, data, content_encoding="gzip", content_length=True): + import email.message + headers = email.message.Message() + if content_encoding is not None: + headers["Content-Encoding"] = content_encoding + if content_length: + headers["Content-Length"] = str(len(data)) + headers["Content-Type"] = "text/plain" + return FakeGzipResponse(data, headers) + + def gzip_bytes(self, data): + import gzip + return gzip.compress(data) + + def opened_request(self, **kwargs): + h = urllib.request.HTTPGzipHandler() + req = Request("http://example.com/", **kwargs) + h.http_request(req) + return h, req + + def test_not_installed_by_default(self): + # Opt-in: build_opener() does not install it, but accepts it explicitly. + o = urllib.request.build_opener() + self.assertFalse(any(isinstance(h, urllib.request.HTTPGzipHandler) + for h in o.handlers)) + o = urllib.request.build_opener(urllib.request.HTTPGzipHandler) + self.assertTrue(any(isinstance(h, urllib.request.HTTPGzipHandler) + for h in o.handlers)) + + def test_request_adds_accept_encoding(self): + h, req = self.opened_request() + self.assertEqual(req.get_header("Accept-encoding"), "gzip") + + def test_request_keeps_user_accept_encoding(self): + h, req = self.opened_request(headers={"Accept-Encoding": "identity"}) + self.assertEqual(req.get_header("Accept-encoding"), "identity") + + def test_response_is_decoded(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), self.body) + + def test_response_headers_stripped(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + info = result.info() + self.assertIsNone(info.get("Content-Encoding")) + self.assertIsNone(info.get("Content-Length")) + self.assertEqual(info.get("Content-Type"), "text/plain") + self.assertEqual(result.status, 200) + result.close() + + def test_response_streamed_in_chunks(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + chunks = [] + while chunk := result.read(64): + self.assertLessEqual(len(chunk), 64) + chunks.append(chunk) + self.assertEqual(b"".join(chunks), self.body) + result.close() + + def test_multiple_gzip_members_decoded(self): + # A response may be several concatenated gzip members; all of them + # must be decoded, not just the first. + h, req = self.opened_request() + comp = self.gzip_bytes(b"AAA") + self.gzip_bytes(b"BBB") + response = self.make_response(comp) + with h.http_response(req, response) as result: + self.assertEqual(result.read(), b"AAABBB") + + def test_user_accept_encoding_passes_through(self): + # A caller-supplied Accept-Encoding is left to the caller to decode: + # the gzipped response is returned untouched. + h, req = self.opened_request(headers={"Accept-Encoding": "gzip"}) + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + self.assertIs(result, response) + self.assertEqual(result.headers.get("Content-Encoding"), "gzip") + + def test_non_gzip_response_passes_through(self): + h, req = self.opened_request() + response = self.make_response(self.body, content_encoding=None) + result = h.http_response(req, response) + self.assertIs(result, response) + + def test_truncated_stream_raises(self): + h, req = self.opened_request() + comp = self.gzip_bytes(self.body) + response = self.make_response(comp[:len(comp) // 2]) + result = h.http_response(req, response) + with self.assertRaises(EOFError): + result.read() + result.close() + + def test_close_propagates(self): + h, req = self.opened_request() + response = self.make_response(self.gzip_bytes(self.body)) + result = h.http_response(req, response) + result.close() + self.assertTrue(response.closed) + + def test_requires_zlib(self): + with mock.patch.dict(sys.modules, {"gzip": None}): + with self.assertRaises(ImportError): + urllib.request.HTTPGzipHandler() + + class MiscTests(unittest.TestCase): def opener_has_handler(self, opener, handler_class): diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py index c33e0aaac15de21..30aeb5fb38754ff 100644 --- a/Lib/urllib/request.py +++ b/Lib/urllib/request.py @@ -84,6 +84,7 @@ import base64 import bisect import contextlib +import copy import email import hashlib import http.client @@ -122,7 +123,7 @@ 'HTTPBasicAuthHandler', 'ProxyBasicAuthHandler', 'AbstractDigestAuthHandler', 'HTTPDigestAuthHandler', 'ProxyDigestAuthHandler', 'HTTPHandler', 'FileHandler', 'FTPHandler', 'CacheFTPHandler', 'DataHandler', - 'UnknownHandler', 'HTTPErrorProcessor', + 'UnknownHandler', 'HTTPErrorProcessor', 'HTTPGzipHandler', # Functions 'urlopen', 'install_opener', 'build_opener', 'pathname2url', 'url2pathname', 'getproxies', @@ -1411,6 +1412,45 @@ def http_response(self, request, response): https_request = http_request https_response = http_response + +class HTTPGzipHandler(BaseHandler): + """Request and transparently decode gzip Content-Encoding responses. + + This handler is not installed by build_opener() by default; add it + explicitly to opt in. It sends Accept-Encoding: gzip unless the request + already carries an Accept-Encoding header, and decodes the response only + when it added that header, so a caller supplying its own Accept-Encoding + still receives the raw response. + """ + + def __init__(self): + # Raise ImportError now, not on the first response, if zlib is missing. + import gzip # noqa: F401 + + def http_request(self, request): + if not request.has_header('Accept-encoding'): + request.add_unredirected_header('Accept-encoding', 'gzip') + request._gzip_injected = True + return request + + def http_response(self, request, response): + if (getattr(request, '_gzip_injected', False) and + response.headers.get('Content-Encoding', '').lower() == 'gzip'): + import gzip + headers = copy.copy(response.headers) + del headers['Content-Encoding'] + del headers['Content-Length'] + fp = addclosehook(gzip.GzipFile(fileobj=response, mode='rb'), + response.close) + result = addinfourl(fp, headers, response.url, response.code) + result.msg = response.msg + return result + return response + + https_request = http_request + https_response = http_response + + class UnknownHandler(BaseHandler): def unknown_open(self, req): type = req.type diff --git a/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst b/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst new file mode 100644 index 000000000000000..bacfaf32f1a8153 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-07-19-12-00-00.gh-issue-43521.aYO2xG.rst @@ -0,0 +1,5 @@ +Add :class:`urllib.request.HTTPGzipHandler`, an opt-in handler that sends an +``Accept-Encoding: gzip`` request header and transparently decodes +gzip-compressed responses. It is not one of the default +:func:`~urllib.request.build_opener` handlers, so it must be added +explicitly.