From 2e32cd1bc155157cdb6c488544bc9e3858c8842e Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:17:18 +0000 Subject: [PATCH 1/8] pubsub: read the effective http defaults back on the client options RTC7 has a client use the timeouts configured in its ClientOptions, falling back to the library defaults, and the UTS test for it reads those defaults back from client.options. The realtime timeouts were already resolved on Options, but http_open_timeout, http_request_timeout, http_max_retry_count and http_max_retry_duration were left as None there and defaulted later, by Http from its CONNECTION_RETRY_DEFAULTS and by the host list from Defaults.http_max_retry_count, so client.options could not say what a client was going to use. The four defaults now live on Defaults and are resolved by Options, the same way as the realtime ones, and Http reads them from the options. CONNECTION_RETRY_DEFAULTS is gone. The values are unchanged and stay in seconds, so the TO3l3 and TO3l4 unit mismatch remains recorded as a deviation in test/uts/deviations.md. This is a breaking change for code that read these options and took None to mean unset: they now read 4, 10, 3 and 15 by default. UPDATING.md says so. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 8 +++++++ ably/pubsub/request/http.py | 23 ++++--------------- ably/pubsub/transport/defaults.py | 4 ++++ ably/pubsub/types/options.py | 15 ++++++++++-- test/ably/http/httphttp_test.py | 6 ++--- test/uts/deviations.md | 14 +++++------ .../unit/client/realtime_timeouts_test.py | 11 ++++----- .../uts/rest/unit/types/options_types_test.py | 12 +++++----- 8 files changed, 49 insertions(+), 44 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 94f4182d..dd8c67f2 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -405,6 +405,14 @@ pubsub_http_client = create_http_client(key='xxx', default_token_params={'capabi await pubsub_http_client.auth.request_token({'ttl': 600000, 'capability': capability}) ``` +### The HTTP options read back their defaults + +`http_open_timeout`, `http_request_timeout`, `http_max_retry_count` and +`http_max_retry_duration` on `client.options` used to read `None` unless you set them, while +the client applied defaults of its own. They now read back the values in effect: 4, 10, 3 and +15 by default. The timeouts and the retry duration are still in seconds. Code that tested an +option for `None`, to tell whether it had been set, will no longer find it so. + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index cd8b54d4..5b69c336 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -126,13 +126,6 @@ def __getattr__(self, attr): class Http: - CONNECTION_RETRY_DEFAULTS = { - 'http_open_timeout': 4, - 'http_request_timeout': 10, - 'http_max_retry_count': 3, - 'http_max_retry_duration': 15, - } - def __init__(self, ably, options): options = options or {} self.__ably = ably @@ -318,24 +311,16 @@ def preferred_scheme(self): @property def http_open_timeout(self): - if self.options.http_open_timeout is not None: - return self.options.http_open_timeout - return self.CONNECTION_RETRY_DEFAULTS['http_open_timeout'] + return self.options.http_open_timeout @property def http_request_timeout(self): - if self.options.http_request_timeout is not None: - return self.options.http_request_timeout - return self.CONNECTION_RETRY_DEFAULTS['http_request_timeout'] + return self.options.http_request_timeout @property def http_max_retry_count(self): - if self.options.http_max_retry_count is not None: - return self.options.http_max_retry_count - return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_count'] + return self.options.http_max_retry_count @property def http_max_retry_duration(self): - if self.options.http_max_retry_duration is not None: - return self.options.http_max_retry_duration - return self.CONNECTION_RETRY_DEFAULTS['http_max_retry_duration'] + return self.options.http_max_retry_duration diff --git a/ably/pubsub/transport/defaults.py b/ably/pubsub/transport/defaults.py index ae5310e6..d239656a 100644 --- a/ably/pubsub/transport/defaults.py +++ b/ably/pubsub/transport/defaults.py @@ -19,7 +19,11 @@ class Defaults: transports = [] # ["web_socket", "comet"] + # The HTTP timeouts are in seconds, unlike the realtime ones above + http_open_timeout = 4 + http_request_timeout = 10 http_max_retry_count = 3 + http_max_retry_duration = 15 fallback_retry_timeout = 600000 # 10min diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 6518744d..76b385b7 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -55,6 +55,19 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti if realtime_request_timeout is None: realtime_request_timeout = Defaults.realtime_request_timeout + # RTC7: the HTTP defaults are resolved here too, so they read back from the options + if http_open_timeout is None: + http_open_timeout = Defaults.http_open_timeout + + if http_request_timeout is None: + http_request_timeout = Defaults.http_request_timeout + + if http_max_retry_count is None: + http_max_retry_count = Defaults.http_max_retry_count + + if http_max_retry_duration is None: + http_max_retry_duration = Defaults.http_max_retry_duration + if disconnected_retry_timeout is None: disconnected_retry_timeout = Defaults.disconnected_retry_timeout @@ -329,8 +342,6 @@ def __get_hosts(self): fallback_hosts = self.get_fallback_hosts() http_max_retry_count = self.http_max_retry_count - if http_max_retry_count is None: - http_max_retry_count = Defaults.http_max_retry_count # Shuffle fallback_hosts = list(fallback_hosts) diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index 91a39146..d2167c3b 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -30,8 +30,8 @@ async def test_host_for_url_brackets_only_ipv6_literals(self): async def test_max_retry_attempts_and_timeouts_defaults(self): ably = create_http_client(token="foo") - assert 'http_open_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS - assert 'http_request_timeout' in ably.http.CONNECTION_RETRY_DEFAULTS + assert ably.options.http_open_timeout == Defaults.http_open_timeout + assert ably.options.http_request_timeout == Defaults.http_request_timeout with mock.patch('httpx.AsyncClient.send', side_effect=httpx.RequestError('')) as send_mock: with pytest.raises(AblyException, match="network error"): @@ -43,7 +43,7 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): async def test_cumulative_timeout(self): ably = create_http_client(token="foo") - assert 'http_max_retry_duration' in ably.http.CONNECTION_RETRY_DEFAULTS + assert ably.options.http_max_retry_duration == Defaults.http_max_retry_duration ably.options.http_max_retry_duration = 0.5 diff --git a/test/uts/deviations.md b/test/uts/deviations.md index bcd2df0f..703f16b0 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1810,7 +1810,7 @@ final response is raised or returned. `ably/http/http.py:193` builds `timeout = (self.http_open_timeout, self.http_request_timeout)` and hands it to `httpx`, which reads both as **seconds**. TO3l4's `httpRequestTimeout` is milliseconds, default 10000, and -`CONNECTION_RETRY_DEFAULTS` holds `10`. So a caller passing the value the published +`Defaults.http_request_timeout` holds `10`. So a caller passing the value the published specification describes gets a deadline a thousand times longer than the one asked for: `http_request_timeout=3000` is three thousand seconds. The **defaults** come out right by coincidence — 10 seconds is TO3l4's 10000 ms, and `http_open_timeout`'s 4 is TO3l3's @@ -1825,9 +1825,9 @@ Passing `3` in its place makes the same test pass in 3.1 seconds: the timeout fi retry goes to the fallback host and succeeds. So RSC15l2's fallback path is compliant and the unit is the whole of the defect. -The same mismatch is recorded twice under *Adapted Tests*, at `TO3l1, TO3l5` and at -`RTC7 (TO3l3, TO3l4)` in *REST behaviours asserted as they are*, where it is what makes -the effective defaults unreadable from `options`. This is that defect seen from outside: +The same mismatch is recorded twice under *Adapted Tests*, at `TO3l4, TO3l5` and at +`RTC7 (TO3l3, TO3l4)` in *REST behaviours asserted as they are*, where it is why the +defaults read back from `options` as 4 and 10. This is that defect seen from outside: the same line of `http.py`, reached through a public client option rather than through an attribute, so a caller is affected whether or not they ever read `options`. It is counted as one root cause, here, because it is the only gated test on it; neither Adapted Tests @@ -1839,7 +1839,7 @@ row is counted again. fixed. **Status:** open bug. The fix converts at the boundary — `Http` dividing the option by -1000 before it reaches `httpx`, with `CONNECTION_RETRY_DEFAULTS` restated in +1000 before it reaches `httpx`, with the `Defaults` restated in milliseconds — and has to move `http_open_timeout` (TO3l3) with it, since both halves of the tuple are read the same way. That same line carries a second, filed defect: [#709](https://github.com/ably/ably-python/issues/709) is that the value, in whatever @@ -2217,8 +2217,8 @@ pseudocode is not mistaken for non-compliance. | CHM2g, CHM2h | `objectPublishers` and `objectSubscribers` on `ChannelMetrics` | Neither is modelled, so both are dropped on parsing. The test asserts their absence, and turns red once they are added | Open bug | | TO3l8 | `maxMessageSize` is a client option, default 65536 | Rejected by `Options.__init__`. `ably/realtime/channel.py:422` reads it with `getattr(..., 65536)`, so the default holds but cannot be configured, nor overridden by `connectionDetails` (CD2c) | Open bug | | RTN15, RTN23 | A DISCONNECTED `ErrorInfo` needs no `statusCode` | `ConnectionManager.on_disconnected` evaluates `exception.status_code >= 500` unguarded, so a DISCONNECTED whose error omits `statusCode` raises `TypeError` in a task whose exception is only logged, and the connection silently stays CONNECTED. `DISCONNECTED_MESSAGE` supplies 400 | Open bug | -| TO3l1, TO3l5 | `httpRequestTimeout` and `httpMaxRetryCount` carry their defaults on the options object | Left unset; the effective defaults are applied downstream by `Http` and by `Options.__get_hosts`. The spec's `httpRequestTimeout` is milliseconds and ably-python's `http_request_timeout` is seconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | Intentional for where the defaults are applied; the unit is an open bug, recorded there | -| RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | Both `None` on `Options`; `Http.http_open_timeout` / `http_request_timeout` fall back to `CONNECTION_RETRY_DEFAULTS`, which holds 4 and 10 — seconds, where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are defaulted on `Options` and match | Two faults in one row: the defaults are unreadable from `options`, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Both open; the unit is gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | +| TO3l4, TO3l5 | `httpRequestTimeout` defaults to 10000 and `httpMaxRetryCount` to 3 on the options object | Both are defaulted on `Options`. `http_max_retry_count` reads back as 3 and matches; `http_request_timeout` reads back as 10, because ably-python's is seconds where the spec's is milliseconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | The unit is an open bug, recorded there | +| RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | 4 and 10: both are defaulted on `Options`, in seconds where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are milliseconds and match | Open bug, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests. Moving the HTTP durations to milliseconds is breaking, and is held back from 4.0.0 pending a decision on units across the options | | RTC1f | a `transportParams` boolean appears as `"true"` / `"false"` | `True` / `False`, because `WebSocketTransport.connect` builds the query string with `urllib.parse.urlencode`, which renders each value through `str()` (`websockettransport.py:89`). Integers are unaffected | Open bug. A caller can pass the strings directly, but a bool is what the spec's Stringifiable type admits | ### LiveObjects: the internal shapes the unit specifications assume — S-1 to S-6 diff --git a/test/uts/realtime/unit/client/realtime_timeouts_test.py b/test/uts/realtime/unit/client/realtime_timeouts_test.py index 3a9c21d9..09da81d4 100644 --- a/test/uts/realtime/unit/client/realtime_timeouts_test.py +++ b/test/uts/realtime/unit/client/realtime_timeouts_test.py @@ -173,10 +173,7 @@ async def test_rtc7_default_timeouts_applied(): assert client.options.suspended_retry_timeout == 30000 # DEVIATION: the spec asserts httpOpenTimeout == 4000 and httpRequestTimeout == - # 10000 on the options. ably-python leaves both unset on the options and holds the - # defaults on the HTTP layer, in seconds rather than milliseconds. See - # deviations.md. - assert client.options.http_open_timeout is None - assert client.options.http_request_timeout is None - assert client.http.http_open_timeout == 4 - assert client.http.http_request_timeout == 10 + # 10000. ably-python's HTTP timeouts are in seconds rather than milliseconds, so the + # same defaults read back as 4 and 10. See deviations.md. + assert client.options.http_open_timeout == 4 + assert client.options.http_request_timeout == 10 diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index bc5848cd..36fe2e4a 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -37,11 +37,11 @@ async def test_to3_client_options_attributes(): assert options.auth_method == 'GET' assert options.tls is True - # DEVIATION: httpRequestTimeout and httpMaxRetryCount have no default on the options - # object; both are left unset and the effective defaults are applied downstream, the - # first by Http and the second by Options.__get_hosts via Defaults.http_max_retry_count. - assert options.http_request_timeout is None - assert options.http_max_retry_count is None + # DEVIATION: the spec asserts httpRequestTimeout == 10000. ably-python's + # http_request_timeout is in seconds rather than milliseconds, so the same default + # reads back as 10. See deviations.md. + assert options.http_request_timeout == 10 + assert options.http_max_retry_count == 3 assert options.use_binary_protocol is True assert options.idempotent_rest_publishing is True @@ -54,7 +54,7 @@ async def test_to3_client_options_attributes(): with pytest.raises(TypeError): Options(max_message_size=65536) - # The effective defaults the options leave unset: 10 seconds, and 3 hosts to try. + # The HTTP layer applies the defaults the options hold: 10 seconds, and 3 hosts to try. client = rest_client(mock_http_client()) assert client.http.http_request_timeout == 10 assert len(client.options.get_hosts()) == 3 From e35b7569e5c7bf43b5636843e8abf6168d42650f Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:19:32 +0000 Subject: [PATCH 2/8] pubsub: remove the per-call timeout argument from requests time(), stats(), the REST channel's publish variants, get_message() on the REST and realtime channels and push.admin.publish() each accepted a timeout keyword argument and passed it down to Http.make_request, which replaced it with the client's (http_open_timeout, http_request_timeout) before building the request. So the argument has never had any effect. stats() also took a paginated argument that nothing read. Both arguments are removed, from the clients, the prototypes and the Http get, post, put, patch and delete helpers and make_request, which no caller needs them for. RTC7 has the timeouts come from the ClientOptions, which is where they were always taken from. This is a breaking change: passing timeout or paginated now raises TypeError. Callers should drop the argument and set http_request_timeout or http_open_timeout on the client instead, as UPDATING.md describes. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 20 ++++++++++++++++++++ ably/pubsub/http/channel.py | 16 ++++++++-------- ably/pubsub/http/http.py | 6 +++--- ably/pubsub/http/push.py | 6 ++---- ably/pubsub/prototypes.py | 4 ++-- ably/pubsub/realtime/channel.py | 6 ++---- ably/pubsub/request/http.py | 22 +++++++++++----------- 7 files changed, 48 insertions(+), 32 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index dd8c67f2..58798e18 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -413,6 +413,26 @@ the client applied defaults of its own. They now read back the values in effect: 15 by default. The timeouts and the retry duration are still in seconds. Code that tested an option for `None`, to tell whether it had been set, will no longer find it so. +### Requests no longer take a `timeout` argument + +`time()`, `stats()`, a REST channel's `publish()`, `publish_message()`, `publish_messages()` +and `publish_name_data()`, `get_message()` on REST and realtime channels, and +`push.admin.publish()` accepted a `timeout` keyword argument that was never applied: every +request used the client's `http_open_timeout` and `http_request_timeout` whatever was passed. +The argument is gone, and passing it raises `TypeError`. `stats()` also loses its unused +`paginated` argument. Set the timeouts on the client instead: + +Example 3.x code: +```python +await rest.time(timeout=5) +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', http_request_timeout=5) +await pubsub_http_client.time() +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index 321ea275..e2bd13a0 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -95,10 +95,10 @@ async def _publish(self, arg, *args, **kwargs): else: raise TypeError(f'Unexpected type {type(arg)}') - async def publish_message(self, message, params=None, timeout=None): - return await self.publish_messages([message], params, timeout=timeout) + async def publish_message(self, message, params=None): + return await self.publish_messages([message], params) - async def publish_messages(self, messages, params=None, timeout=None): + async def publish_messages(self, messages, params=None): request_body = self.__publish_request_body(messages) if not self.ably.options.use_binary_protocol: request_body = json.dumps(request_body, separators=(',', ':')) @@ -109,7 +109,7 @@ async def publish_messages(self, messages, params=None, timeout=None): if params: params = {k: str(v).lower() if isinstance(v, bool) else v for k, v in params.items()} path += '?' + parse.urlencode(params) - response = await self.ably.http.post(path, body=request_body, timeout=timeout) + response = await self.ably.http.post(path, body=request_body) # Parse response to extract serials result_data = response.to_native() @@ -117,9 +117,9 @@ async def publish_messages(self, messages, params=None, timeout=None): return PublishResult.from_dict(result_data) return PublishResult() - async def publish_name_data(self, name, data, timeout=None): + async def publish_name_data(self, name, data): messages = [Message(name, data)] - return await self.publish_messages(messages, timeout=timeout) + return await self.publish_messages(messages) async def publish(self, *args, **kwargs): """Publishes a message on this channel. @@ -259,7 +259,7 @@ async def append_message(self, message: Message, operation: MessageOperation = N """ return await self._send_update(message, MessageAction.MESSAGE_APPEND, operation, params) - async def get_message(self, serial_or_message, timeout=None): + async def get_message(self, serial_or_message): """Retrieves a single message by its serial. Parameters: @@ -291,7 +291,7 @@ async def get_message(self, serial_or_message, timeout=None): path = self.__base_path + 'messages/' + parse.quote_plus(serial, safe=':') # Make the request - response = await self.ably.http.get(path, timeout=timeout) + response = await self.ably.http.get(path) # Create Message from the response message_handler = make_single_message_response_handler(self.__cipher) diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index 5c14f7bd..dd738c1a 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -103,7 +103,7 @@ async def __aenter__(self): @catch_all async def stats(self, direction: Optional[str] = None, start=None, end=None, params: Optional[dict] = None, - limit: Optional[int] = None, paginated=None, unit=None, timeout=None): + limit: Optional[int] = None, unit=None): """Returns the stats for this application""" formatted_params = format_params(params, direction=direction, start=start, end=end, limit=limit, unit=unit) url = '/stats' + formatted_params @@ -111,9 +111,9 @@ async def stats(self, direction: Optional[str] = None, start=None, end=None, par self.http, url=url, response_processor=stats_response_processor) @catch_all - async def time(self, timeout: Optional[float] = None) -> float: + async def time(self) -> float: """Returns the current server time in ms since the unix epoch""" - r = await self.http.get('/time', skip_auth=True, timeout=timeout) + r = await self.http.get('/time', skip_auth=True) AblyException.raise_for_response(r) return r.to_native()[0] diff --git a/ably/pubsub/http/push.py b/ably/pubsub/http/push.py index a127de1f..1fc6ea98 100644 --- a/ably/pubsub/http/push.py +++ b/ably/pubsub/http/push.py @@ -1,5 +1,3 @@ -from typing import Optional - from ably.pubsub.request.paginatedresult import PaginatedResult, format_params from ably.pubsub.types.channelsubscription import ( PushChannelSubscription, @@ -40,7 +38,7 @@ def device_registrations(self): def channel_subscriptions(self): return self.__channel_subscriptions - async def publish(self, recipient: dict, data: dict, timeout: Optional[float] = None): + async def publish(self, recipient: dict, data: dict): """Publish a push notification to a single device. :Parameters: @@ -61,7 +59,7 @@ async def publish(self, recipient: dict, data: dict, timeout: Optional[float] = body = data.copy() body.update({'recipient': recipient}) - await self.ably.http.post('/push/publish', body=body, timeout=timeout) + await self.ably.http.post('/push/publish', body=body) class PushDeviceRegistrations: diff --git a/ably/pubsub/prototypes.py b/ably/pubsub/prototypes.py index 4fc2613e..3ec07b9d 100644 --- a/ably/pubsub/prototypes.py +++ b/ably/pubsub/prototypes.py @@ -65,11 +65,11 @@ def push(self) -> Push: ... async def stats(self, direction: str | None = None, start=None, end=None, params: dict | None = None, - limit: int | None = None, paginated=None, unit=None, timeout=None) -> PaginatedResult: + limit: int | None = None, unit=None) -> PaginatedResult: """Return the stats for this application.""" ... - async def time(self, timeout: float | None = None) -> float: + async def time(self) -> float: """Return the current server time in ms since the unix epoch.""" ... diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 17796cf2..7da4ed1f 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -689,15 +689,13 @@ async def append_message( """ return await self._send_update(message, MessageAction.MESSAGE_APPEND, operation, params) - async def get_message(self, serial_or_message, timeout=None): + async def get_message(self, serial_or_message): """Retrieves a single message by its serial using the REST API. Parameters ---------- serial_or_message : str or Message Either a string serial or a Message object with a serial field. - timeout : float, optional - Timeout for the request. Returns ------- @@ -710,7 +708,7 @@ async def get_message(self, serial_or_message, timeout=None): If the serial is missing or the message cannot be retrieved. """ # Delegate to parent Channel (REST) implementation - return await Channel.get_message(self, serial_or_message, timeout=timeout) + return await Channel.get_message(self, serial_or_message) async def get_message_versions(self, serial_or_message, params=None): """Retrieves version history for a message using the REST API. diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index 5b69c336..8118d659 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -172,7 +172,7 @@ def get_hosts(self): @reauth_if_expired async def make_request(self, method, path, version=None, headers=None, body=None, - skip_auth=False, timeout=None, raise_on_error=True): + skip_auth=False, raise_on_error=True): if body is not None and type(body) not in (bytes, str): body = self.dump_body(body) @@ -260,29 +260,29 @@ async def request_external(self, method, url, headers=None, params=None, body=No """ return await self.__client.request(method=method, url=url, headers=headers, params=params, data=body) - async def delete(self, url, headers=None, skip_auth=False, timeout=None): + async def delete(self, url, headers=None, skip_auth=False): result = await self.make_request('DELETE', url, headers=headers, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def get(self, url, headers=None, skip_auth=False, timeout=None): + async def get(self, url, headers=None, skip_auth=False): result = await self.make_request('GET', url, headers=headers, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def patch(self, url, headers=None, body=None, skip_auth=False, timeout=None): + async def patch(self, url, headers=None, body=None, skip_auth=False): result = await self.make_request('PATCH', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def post(self, url, headers=None, body=None, skip_auth=False, timeout=None): + async def post(self, url, headers=None, body=None, skip_auth=False): result = await self.make_request('POST', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result - async def put(self, url, headers=None, body=None, skip_auth=False, timeout=None): + async def put(self, url, headers=None, body=None, skip_auth=False): result = await self.make_request('PUT', url, headers=headers, body=body, - skip_auth=skip_auth, timeout=timeout) + skip_auth=skip_auth) return result @property From 5b8be73e25961aa8b905dd3ccf52c3c7a4b9419b Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:22:07 +0000 Subject: [PATCH 3/8] pubsub: bound the write and pool phases of an http request Http.make_request passed httpx the tuple (http_open_timeout, http_request_timeout), which httpx expands to a connect and a read timeout and leaves write and pool as None. Writing a request body to a wedged connection, or waiting for a free pooled connection, therefore never timed out, whatever the client's options said. This is one of the three parts of issue #709. The timeout is now an explicit httpx.Timeout: connect takes http_open_timeout (TO3l3), and read, write and pool take http_request_timeout (TO3l4). The other parts of #709, that the read timeout applies per socket read and that the RSC15 retry budget is not consulted while an attempt is in flight, are left as they are, as is the unit of both options, which stays seconds. This is a breaking change in behaviour: a request that used to hang in either phase now raises httpx.TimeoutException after http_request_timeout and is retried against a fallback host. Callers who send large bodies over slow links may need to raise http_request_timeout. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 13 +++++++++++++ ably/pubsub/request/http.py | 3 ++- test/unit/http_test.py | 12 ++++++++++++ test/uts/deviations.md | 21 ++++++++++++--------- 4 files changed, 39 insertions(+), 10 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 58798e18..9db3867e 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -433,6 +433,19 @@ pubsub_http_client = create_http_client(key='xxx', http_request_timeout=5) await pubsub_http_client.time() ``` +### `http_request_timeout` bounds writes and connection-pool waits + +In 3.x, `http_open_timeout` bounded connecting and `http_request_timeout` bounded each read +of a response, while writing a request body and waiting for a free pooled connection had no +timeout at all. `http_request_timeout` now bounds those too, so a request stuck writing to, +or waiting for, a wedged connection raises `httpx.TimeoutException` and is retried against a +fallback host instead of hanging. If you publish large bodies over a slow link, raise +`http_request_timeout` to fit: + +```python +pubsub_http_client = create_http_client(key='xxx', http_request_timeout=30) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/request/http.py b/ably/pubsub/request/http.py index 8118d659..9e19e6e0 100644 --- a/ably/pubsub/request/http.py +++ b/ably/pubsub/request/http.py @@ -195,7 +195,8 @@ async def make_request(self, method, path, version=None, headers=None, body=None if headers: all_headers.update(headers) - timeout = (self.http_open_timeout, self.http_request_timeout) + # TO3l3 bounds opening the connection, and TO3l4 each read, write and wait for a pooled connection + timeout = httpx.Timeout(self.http_request_timeout, connect=self.http_open_timeout) http_max_retry_duration = self.http_max_retry_duration requested_at = self.__clock.now_ms() diff --git a/test/unit/http_test.py b/test/unit/http_test.py index c5be5803..70608c81 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -51,6 +51,18 @@ async def test_http_sends_requests_through_an_injected_transport(): await ably.close() +async def test_http_bounds_every_phase_of_a_request(): + transport = RecordingTransport(lambda request: httpx.Response(200, json=[1500000000000])) + ably = create_http_client(token="foo", http_open_timeout=2, http_request_timeout=7, + _test_options=TestOptions(http_transport=transport)) + + await ably.time() + + # TO3l3 bounds the connect, and TO3l4 the read, the write and the wait for a pooled connection + assert transport.requests[0].extensions['timeout'] == {'connect': 2, 'read': 7, 'write': 7, 'pool': 7} + await ably.close() + + async def test_http_surfaces_transport_connection_errors(): def refuse(request): raise httpx.ConnectError("connection refused", request=request) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 703f16b0..ab73ca78 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1807,8 +1807,9 @@ final response is raised or returned. **Spec points:** TO3l4, RSC15l2, `rest/proxy/RSC15l2/timeout-triggers-fallback-0`. -`ably/http/http.py:193` builds `timeout = (self.http_open_timeout, -self.http_request_timeout)` and hands it to `httpx`, which reads both as **seconds**. +`Http.make_request` (`ably/pubsub/request/http.py`) builds `timeout = +httpx.Timeout(self.http_request_timeout, connect=self.http_open_timeout)` and hands it +to `httpx`, which reads both as **seconds**. TO3l4's `httpRequestTimeout` is milliseconds, default 10000, and `Defaults.http_request_timeout` holds `10`. So a caller passing the value the published specification describes gets a deadline a thousand times longer than the one asked for: @@ -1840,13 +1841,15 @@ fixed. **Status:** open bug. The fix converts at the boundary — `Http` dividing the option by 1000 before it reaches `httpx`, with the `Defaults` restated in -milliseconds — and has to move `http_open_timeout` (TO3l3) with it, since both halves of -the tuple are read the same way. That same line carries a second, filed defect: +milliseconds — and has to move `http_open_timeout` (TO3l3) with it, since both are +read the same way. Moving the unit is breaking, and is held back from 4.0.0 pending a +decision on units across the options. That same line carries a second, filed defect: [#709](https://github.com/ably/ably-python/issues/709) is that the value, in whatever unit, bounds one socket read rather than the request, so a connection that keeps -producing frames is never timed out and `write` and `pool` are left unbounded. The two -want fixing together, since both change what one attempt may spend of the RSC15 retry -budget. +producing frames is never timed out. Of #709's three parts, only the unbounded `write` +and `pool` timeouts are fixed, both now taking `http_request_timeout`. The rest of it and +the unit want fixing together, since both change what one attempt may spend of the +RSC15 retry budget. ### Requests @@ -2910,8 +2913,8 @@ the two meanings `raise_on_error` carries. `test/uts/rest/integration/auth_test.py -k rsc10` **I.3 `httpRequestTimeout` is applied as seconds where the specification counts -milliseconds.** TO3l4, TO3l3, RSC15l2. `ably/http/http.py:193` hands -`(http_open_timeout, http_request_timeout)` to `httpx`, which reads seconds, so a client +milliseconds.** TO3l4, TO3l3, RSC15l2. `Http.make_request` hands `http_open_timeout` +and `http_request_timeout` to `httpx`, which reads seconds, so a client built with the specification's `httpRequestTimeout: 3000` waits three thousand seconds. Measured through `uts-proxy` against a session delaying `/time` by 20 seconds: the request sat out the whole delay, succeeded on the primary host and attempted no From 42d83c2b966b18723cd260ecad72d7ad1cbee9df Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:26:32 +0000 Subject: [PATCH 4/8] pubsub: count only fallback hosts against http_max_retry_count TO3l5 defines httpMaxRetryCount as the maximum number of fallback hosts to use when a request to the primary host fails. Options built the host list by truncating the primary plus the fallbacks to http_max_retry_count, so it counted the primary too: the default of 3 tried the primary and two fallbacks, one host fewer than ably-js, and http_max_retry_count=0 left the list empty, so every request failed with an AttributeError wrapped as a 50000. The host list is now the primary followed by at most http_max_retry_count of the shuffled fallback hosts, so the default tries four hosts and 0 tries the primary alone. RSC15a's row in test/uts/deviations.md called the old truncation intentional; it now records the four hosts as compliant. This is a breaking change: a client tries one more host per request for the same setting. Callers who want the 3.x number of attempts should pass one less, as UPDATING.md shows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 20 +++++++++++++++++++ ably/pubsub/server/__init__.py | 3 ++- ably/pubsub/types/options.py | 10 +++------- test/ably/http/httphttp_test.py | 7 ++++--- test/unit/http_test.py | 18 +++++++++++++++++ test/uts/deviations.md | 2 +- test/uts/rest/unit/fallback_test.py | 8 ++++---- test/uts/rest/unit/request_test.py | 4 ++-- .../uts/rest/unit/types/options_types_test.py | 5 +++-- 9 files changed, 57 insertions(+), 20 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 9db3867e..444a265f 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -446,6 +446,26 @@ fallback host instead of hanging. If you publish large bodies over a slow link, pubsub_http_client = create_http_client(key='xxx', http_request_timeout=30) ``` +### `http_max_retry_count` counts fallback hosts only + +In 3.x, `http_max_retry_count` capped the total number of hosts a request was tried +against, the primary endpoint included, so the default of 3 meant the primary and two +fallbacks. It now counts fallback hosts only, as in the Ably specification and the other +Ably SDKs: a request goes to the primary endpoint and then to at most +`http_max_retry_count` fallback hosts. The default of 3 therefore tries one more host than +before, and `http_max_retry_count=0`, which used to fail every request, now tries the +primary endpoint alone. To keep the 3.x number of attempts, subtract one: + +Example 3.x code: +```python +rest = AblyRest(key='xxx', http_max_retry_count=3) +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', http_max_retry_count=2) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index cb75bba7..8d2da1f0 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -136,7 +136,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: How long (in seconds) the client keeps retrying a request across hosts before giving up. The default is 15 seconds. http_max_retry_count: int - The most hosts the client will try for a single request. + The most fallback hosts the client will try for a single request, after the + primary endpoint. The default is 3; 0 tries the primary endpoint only. **Protocol** diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 76b385b7..ae448e49 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -335,23 +335,19 @@ def __get_hosts(self): """ Return the list of hosts as they should be tried. First comes the main host. Then the fallback hosts in random order. - The returned list will have a length of up to http_max_retry_count. + The returned list will hold up to http_max_retry_count fallback hosts. """ host = Defaults.get_hostname(self.endpoint) # REC2: Determine fallback hosts fallback_hosts = self.get_fallback_hosts() - http_max_retry_count = self.http_max_retry_count - # Shuffle fallback_hosts = list(fallback_hosts) random.shuffle(fallback_hosts) self.__fallback_hosts = fallback_hosts - # First main host - hosts = [host] + fallback_hosts - hosts = hosts[:http_max_retry_count] - return hosts + # First main host, then TO3l5: http_max_retry_count counts the fallback hosts only + return [host] + fallback_hosts[:self.http_max_retry_count] def get_hosts(self): return self.__hosts diff --git a/test/ably/http/httphttp_test.py b/test/ably/http/httphttp_test.py index d2167c3b..0336d0ef 100644 --- a/test/ably/http/httphttp_test.py +++ b/test/ably/http/httphttp_test.py @@ -37,7 +37,8 @@ async def test_max_retry_attempts_and_timeouts_defaults(self): with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', version=Defaults.protocol_version, skip_auth=True) - assert send_mock.call_count == Defaults.http_max_retry_count + # TO3l5: the primary host, then http_max_retry_count fallbacks + assert send_mock.call_count == Defaults.http_max_retry_count + 1 assert send_mock.call_args == mock.call(mock.ANY) await ably.close() @@ -70,7 +71,7 @@ def make_url(host): with pytest.raises(AblyException, match="network error"): await ably.http.make_request('GET', '/', skip_auth=True) - assert send_mock.call_count == Defaults.http_max_retry_count + assert send_mock.call_count == Defaults.http_max_retry_count + 1 expected_urls_set = { make_url(host) @@ -169,7 +170,7 @@ async def test_500_errors(self): with pytest.raises(AblyException): await ably.http.make_request('GET', '/', skip_auth=True) - assert mock_request.call_count == 3 + assert mock_request.call_count == 4 await ably.close() diff --git a/test/unit/http_test.py b/test/unit/http_test.py index 70608c81..91960630 100644 --- a/test/unit/http_test.py +++ b/test/unit/http_test.py @@ -99,6 +99,24 @@ def time_out(request): await ably.close() +async def test_http_tries_the_primary_and_up_to_max_retry_count_fallbacks(): + def refuse(request): + raise httpx.ConnectError("connection refused", request=request) + + # TO3l5: http_max_retry_count counts the fallback hosts, after the primary + for retry_count, attempts in [(0, 1), (2, 3), (3, 4), (10, 6)]: + transport = RecordingTransport(refuse) + ably = create_http_client(token="foo", http_max_retry_count=retry_count, + _test_options=TestOptions(http_transport=transport)) + + with pytest.raises(AblyException): + await ably.time() + + assert len(transport.requests) == attempts, retry_count + assert transport.requests[0].url.host == ably.options.get_host() + await ably.close() + + def test_http_uses_a_network_transport_without_test_options(): ably = create_http_client(token="foo") assert isinstance(ably.http._Http__client._transport, httpx.AsyncHTTPTransport) diff --git a/test/uts/deviations.md b/test/uts/deviations.md index ab73ca78..f0412b45 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -2212,7 +2212,7 @@ pseudocode is not mistaken for non-compliance. | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | | HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | | HP8 | `headers` is a map | A list of `(name, value)` pairs, so the lookup the spec describes is impossible without converting, and case-insensitivity is lost | Open bug; changing the return type is breaking | -| RSC15a | Six hosts tried | Three. `Options.__get_hosts` truncates to `http_max_retry_count`, which TO3l5 sanctions | Intentional | +| RSC15a | Six hosts tried | Four: the primary and three fallbacks, because TO3l5's `httpMaxRetryCount`, default 3, caps the fallback hosts tried. The specification's six is the number of responses its setup queues, with a comment counting the primary and all five fallbacks, and no assertion reads it | Compliant; the setup's comment overlooks TO3l5 | | TI | `ErrorInfo` equality by attributes | No `__eq__`, so errors compare by identity. Python exceptions conventionally do, and the requirement appears nowhere in `features.md`. Adding `__eq__` without `__hash__` would make `AblyException` unhashable and break any caller that puts one in a set | Intentional | | RSA6b, RSA6d | The capability literal as passed | Canonicalised by `Capability.c14n`, which RSA9f requires | Compliant; rendering only | | RSA8d | `error.message` contains the callback's text | Wrapped as 40170 with the original in `cause`; `__str__` renders both | Rendering | diff --git a/test/uts/rest/unit/fallback_test.py b/test/uts/rest/unit/fallback_test.py index 3cc8eafd..d5f77a38 100644 --- a/test/uts/rest/unit/fallback_test.py +++ b/test/uts/rest/unit/fallback_test.py @@ -82,10 +82,10 @@ async def test_rsc15a_fallback_random_order(): assert all(host in DEFAULT_FALLBACK_HOSTS for host in fallback_hosts_used) assert len(set(fallback_hosts_used)) == len(fallback_hosts_used) - # NOTE: the spec queues six responses, for the primary and all five fallbacks. - # ably-python truncates the host list to http_max_retry_count (3 by default), so - # the primary and two fallbacks are tried. - assert len(requests) == 3 + # NOTE: the spec queues six responses, for the primary and all five fallbacks, and + # asserts no count. TO3l5's http_max_retry_count (3 by default) caps the fallbacks + # tried, so the primary and three fallbacks are. + assert len(requests) == 4 # The spec leaves verifying randomness to the implementation. Ordering is drawn once # per client, so several clients are built and their host orders compared. diff --git a/test/uts/rest/unit/request_test.py b/test/uts/rest/unit/request_test.py index a1bd7a47..20132fa7 100644 --- a/test/uts/rest/unit/request_test.py +++ b/test/uts/rest/unit/request_test.py @@ -19,8 +19,8 @@ SPEC_KEY = 'appId.keyId:keySecret' # The number of hosts a single request is tried against: the primary domain plus -# fallbacks, capped at Defaults.http_max_retry_count -HOST_ATTEMPTS = 3 +# up to Defaults.http_max_retry_count fallbacks (TO3l5) +HOST_ATTEMPTS = 4 def succeed(conn): diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index 36fe2e4a..471b940a 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -54,10 +54,11 @@ async def test_to3_client_options_attributes(): with pytest.raises(TypeError): Options(max_message_size=65536) - # The HTTP layer applies the defaults the options hold: 10 seconds, and 3 hosts to try. + # The HTTP layer applies the defaults the options hold: 10 seconds, and the primary + # host plus 3 fallbacks to try (TO3l5). client = rest_client(mock_http_client()) assert client.http.http_request_timeout == 10 - assert len(client.options.get_hosts()) == 3 + assert len(client.options.get_hosts()) == 4 options = Options( key='appId.keyId:keySecret', From 4aca1bdb2945e0543e2cbc0fc0319eb1c542004f Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:29:07 +0000 Subject: [PATCH 5/8] pubsub: remove the log_level client option TO3b leaves the implementation of logLevel to the platform. ably-python accepted a log_level option, stored it on Options and gave it a property and a setter, but nothing read it, so setting it changed nothing. The library has always logged through the standard logging module, under the ably.pubsub logger, which is where Python applications expect to set a level or attach a handler. The option is removed, and the factory docstrings now say how to configure the ably.pubsub logger instead. No log handler option is added. The RSC2, RSC3, RSC4, TO3b, TO3c and TO3c2 tests stay gated, and their row in test/uts/deviations.md now records that neither option exists. This is a breaking change: passing log_level now raises TypeError. Callers should drop it and configure logging.getLogger('ably.pubsub') instead, as UPDATING.md shows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 23 +++++++++++++++++++++++ ably/pubsub/server/__init__.py | 8 ++++++-- ably/pubsub/server/sync.py | 1 + ably/pubsub/types/options.py | 11 +---------- test/uts/deviations.md | 2 +- test/uts/rest/unit/logging_test.py | 18 ++++++++++-------- 6 files changed, 42 insertions(+), 21 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 444a265f..827c087b 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -466,6 +466,29 @@ Example 4.0.0 code: pubsub_http_client = create_http_client(key='xxx', http_max_retry_count=2) ``` +### The `log_level` option is removed + +`log_level` was stored on the client options and never read, so it changed nothing. It is +gone, and passing it raises `TypeError`. The library logs through Python's standard +`logging` module, to the `ably.pubsub` logger and the loggers beneath it, so configure that +logger instead to set the level or to choose where records go: + +Example 3.x code: +```python +rest = AblyRest(key='xxx', log_level=logging.DEBUG) +``` + +Example 4.0.0 code: +```python +import logging + +logger = logging.getLogger('ably.pubsub') +logger.setLevel(logging.DEBUG) +logger.addHandler(logging.StreamHandler()) + +pubsub_http_client = create_http_client(key='xxx') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 8d2da1f0..55a1cf9f 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -60,6 +60,11 @@ def create_http_client(**kwargs) -> PubSubHttpClient: of `key`, `token` or `token_details` to authenticate, or configure token auth through `auth_callback` or `auth_url`. + The client logs through the standard :mod:`logging` module, to the `ably.pubsub` logger + and the loggers beneath it, and adds no handler of its own beyond a `NullHandler`. + Configure that logger to choose the level and where records go, for example + ``logging.getLogger('ably.pubsub').setLevel(logging.DEBUG)``. + Parameters ---------- **kwargs: client options @@ -149,8 +154,6 @@ def create_http_client(**kwargs) -> PubSubHttpClient: add_request_ids: bool Add a `request_id` query parameter to every request, which makes a request traceable in Ably's logs. Defaults to False. - log_level: int - Verbosity of the library's own logging. Defaults to 0. **Deprecated** @@ -183,6 +186,7 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: Everything the HTTP client does, plus subscribing to channels and entering presence. Accepts every option :func:`create_http_client` does — see there for credentials, token authentication, connection and HTTP behaviour — and the following in addition. + It logs to the `ably.pubsub` logger, as :func:`create_http_client` describes. Parameters ---------- diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index 56357a6e..c5d8cbe1 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -44,6 +44,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Accepts every option :func:`ably.pubsub.server.create_http_client` does — see there for credentials, token authentication, connection and HTTP behaviour — and behaves identically, without the event loop. The realtime-only options have no effect here. + It logs to the `ably.pubsub` logger, as that function describes. Returns ------- diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index ae448e49..e33bbb97 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -26,7 +26,7 @@ def decode(self, delta: bytes, base: bytes) -> bytes: class Options(AuthOptions): - def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realtime_host=None, port=0, + def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, port=0, tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None, environment=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, @@ -115,7 +115,6 @@ def __init__(self, client_id=None, log_level=0, tls=True, rest_host=None, realti endpoint = Defaults.endpoint self.__client_id = client_id - self.__log_level = log_level self.__tls = tls self.__port = port self.__tls_port = tls_port @@ -155,14 +154,6 @@ def client_id(self): def client_id(self, value): self.__client_id = value - @property - def log_level(self): - return self.__log_level - - @log_level.setter - def log_level(self, value): - self.__log_level = value - @property def tls(self): return self.__tls diff --git a/test/uts/deviations.md b/test/uts/deviations.md index f0412b45..a607c6d3 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -1044,7 +1044,7 @@ on the `recover` parameter the connection never sends. | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | Derived channels: `DeriveOptions` and `Channels.getDerived`. `grep -r derive ably/` is empty. Each test imports `DeriveOptions` inside its body so the module still loads | 5 | | RTB1, RTB1a, RTB1b | Retry backoff, jitter and `retryIn`. Retry timers schedule the flat configured timeout (`connectionmanager.py:753`, `channel.py:866-871`); `grep` for jitter/backoff/retry_in returns nothing, and neither `ConnectionStateChange` nor `ChannelStateChange` carries `retryIn` | 4 | | RTL25, RTL25a, RTL25b | `RealtimeChannel#whenState`. `Connection._when_state` exists (private, awaitable), so this is a gap on the channel rather than a house style; the tests are written against a `channel.when_state(state)` matching the shape the connection already has | 4 | -| RSC2, RSC3, RSC4, TO3b, TO3c, TO3c2 | `log_handler` as a client option, and any use of `log_level` — it is stored on `Options` and read by nothing | 4 | +| RSC2, RSC3, RSC4, TO3b, TO3c, TO3c2 | `log_level` and `log_handler` as client options. The library logs through the standard `logging` module under the `ably.pubsub` logger, whose level and handlers are the idiomatic way to do both; `log_level`, which was stored on `Options` and read by nothing, was removed in 4.0.0, so passing either raises `TypeError` | 4 | | RSP3a2, RSP3a3 | `clientId` and `connectionId` filters on `RestPresence#get`. `Presence.get` is `get(self, limit=None)` (`ably/types/presence.py:216`), while `Presence.history` does take its documented params. `presence.get(client_id=...)` raises `TypeError: get() got an unexpected keyword argument 'client_id'`. The absence also forces the RSP5 decoding adaptation below | 4 (5 cases) | | TP3a, TP3d, TP3g | Presence attributes defaulted from the encapsulating ProtocolMessage. There is no ProtocolMessage type; `ably/realtime/channel.py:751-761` passes the presence array through without context. Matters for synthesized-leave detection and `memberKey` | 3 | | TB4, RTL7h, RTP6e | `attachOnSubscribe`. `ChannelOptions.__init__` (`channeloptions.py:22-26`) takes only `cipher`, `params` and `modes`, and `subscribe()` on the channel, on presence and on annotations all end unconditionally with `await attach()`. This absence also forces the largest single adaptation in the suite, below | 3 | diff --git a/test/uts/rest/unit/logging_test.py b/test/uts/rest/unit/logging_test.py index 21984752..47b8d3c7 100644 --- a/test/uts/rest/unit/logging_test.py +++ b/test/uts/rest/unit/logging_test.py @@ -2,10 +2,11 @@ Spec points: RSC2, RSC2b, RSC3, RSC4, TO3b, TO3c, TO3c2 -ably-python has no log_handler client option, and logs through the standard library -`logging` module: a handler attached to the `ably` logger is the idiomatic rendering of -the specification's `logHandler`. The `log_level` option exists but is never applied to -any logger, so the tests that turn on verbosity or silence it depart from the spec. +ably-python has neither a log_level nor a log_handler client option, and logs through the +standard library `logging` module, under the `ably.pubsub` logger: setting that logger's +level, or attaching a handler to it, is the idiomatic rendering of the specification's +`logLevel` and `logHandler`. The tests that set either as a client option depart from the +spec. """ import logging @@ -79,8 +80,8 @@ async def test_rsc2_default_log_level_warn(): # UTS: rest/unit/TO3b/log-level-changeable-0 @deviation async def test_to3b_log_level_changeable(): - # DEVIATION: log_level is accepted and stored, but never applied to a logger, so raising - # it to verbose emits nothing. The SDK also has no "HTTP request" log event. + # DEVIATION: there is no log_level client option, so this raises TypeError on + # construction. The SDK also has no "HTTP request" log event. captured_logs = [] mock_http = MockHttpClient(on_connection_attempt=connect_successfully, on_request=time_responder) @@ -143,8 +144,9 @@ def handler(level, message, context): # UTS: rest/unit/RSC2b/log-level-none-suppresses-all-0 @deviation async def test_rsc2b_log_level_none_suppresses_all(): - # DEVIATION: log_level is never applied, so it silences nothing. The SDK's logger is - # turned up to debug first, so that suppression is observable at all. + # DEVIATION: there is no log_level client option, so this raises TypeError on + # construction. The SDK's logger is turned up to debug first, so that suppression + # would be observable at all. captured_logs = [] mock_http = MockHttpClient(on_connection_attempt=connect_successfully, on_request=time_responder) From 50f7c334f0d0806da87964abfd4953216ec96c9a Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:33:38 +0000 Subject: [PATCH 6/8] pubsub: remove the deprecated and duplicate client options environment, rest_host and realtime_host were deprecated in favour of endpoint in 3.0.0, and TO3k1, TO3k2 and TO3k3 let a library with no backwards-compatibility requirement omit them. Options still accepted all three, mapped them onto endpoint with a warning, and carried the REC1b1 and REC1c1 conflict checks between them. The client options also took the token under two names, token (TO3j2), which the factories document, and auth_token, which the http client used internally and passed through from callers unchecked. And Options had a connection_state_ttl property and setter that copied Defaults.connection_state_ttl and that nothing read; the realtime client's docstring documented a connection_state_ttl keyword the constructor has never accepted. All of these are removed, along with the docstrings that mentioned them. endpoint is the one way to name a host, and the token option is token, read back as options.token. The UTS tests for REC1b1, REC1c, REC1d, REC2c5 and REC2c6 that exist only to exercise the removed host options are skipped as not applicable, and the four that used one only to name a host use endpoint instead; test/uts/deviations.md records both and updates its counts. This is a breaking change: passing environment, rest_host, realtime_host or auth_token now raises TypeError, and options.auth_token and options.connection_state_ttl are gone. Callers should pass endpoint for the first three and token for the fourth, as UPDATING.md shows. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 37 ++++++++-- ably/pubsub/http/auth.py | 8 +- ably/pubsub/http/http.py | 9 +-- ably/pubsub/realtime/realtime.py | 10 --- ably/pubsub/server/__init__.py | 25 ++----- ably/pubsub/server/sync.py | 5 +- ably/pubsub/types/authoptions.py | 15 ++-- ably/pubsub/types/options.py | 55 +------------- test/ably/http/httpinit_test.py | 12 +-- test/unit/options_test.py | 74 ++----------------- test/unit/websockettransport_test.py | 2 +- test/uts/deviations.md | 36 ++++++--- .../unit/connection/fallback_hosts_test.py | 5 +- test/uts/rest/unit/fallback_test.py | 19 +++++ .../uts/rest/unit/types/options_types_test.py | 11 +-- .../rest/unit/types/paginated_result_test.py | 3 +- 16 files changed, 127 insertions(+), 199 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 827c087b..9b609914 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -196,11 +196,9 @@ async def auth_callback(token_params): The factories raised a bare `ValueError` in 3.x when given no means of authenticating, a `key` together with `key_name` or `key_secret`, a `token_details` that is not a `TokenDetails`, or `use_token_auth=False` without a key. In 4.0.0 each raises an -`AblyException` with code 40106 and status 401. Combining `endpoint` with `environment`, -`rest_host` or `realtime_host` still raises an `AblyException` with code 40106 and status -400. `push.admin.publish` raised `TypeError` for a `recipient` or `data` that is not a dict -and `ValueError` for an empty one, and now raises an `AblyException` with code 40000 and -status 400 for all four. +`AblyException` with code 40106 and status 401. `push.admin.publish` raised `TypeError` +for a `recipient` or `data` that is not a dict and `ValueError` for an empty one, and now +raises an `AblyException` with code 40000 and status 400 for all four. These exceptions are also a `ValueError`, so `except ValueError` still catches them, but `except TypeError` no longer catches the `push.admin.publish` ones: @@ -489,6 +487,35 @@ logger.addHandler(logging.StreamHandler()) pubsub_http_client = create_http_client(key='xxx') ``` +### Deprecated and duplicate options are removed + +These options were deprecated in 3.x, or duplicated another option, and are gone in 4.0.0. +Passing any of them raises `TypeError`: + +- `environment`: pass the same value as `endpoint`, so `environment='sandbox'` becomes + `endpoint='sandbox'`. +- `rest_host` and `realtime_host`: pass the host as `endpoint`, which one host now serves + for both REST and realtime. +- `auth_token`: pass the token as `token`, the name the factories document. The token is now + read back as `client.options.token` rather than `client.options.auth_token`. +- `connection_state_ttl`: this was never accepted as an option, and the + `client.options.connection_state_ttl` property, which nothing read, is removed. The value + the server sends is on `connection.connection_details.connection_state_ttl`. + +Example 3.x code: +```python +rest = AblyRest(key='xxx', environment='sandbox') +realtime = AblyRealtime(key='xxx', realtime_host='custom.ably.example.com') +rest = AblyRest(auth_token='xxx') +``` + +Example 4.0.0 code: +```python +pubsub_http_client = create_http_client(key='xxx', endpoint='sandbox') +pubsub_realtime_client = create_realtime_client(key='xxx', endpoint='custom.ably.example.com') +pubsub_http_client = create_http_client(token='xxx') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/auth.py b/ably/pubsub/http/auth.py index 106e10c8..3f0399e7 100644 --- a/ably/pubsub/http/auth.py +++ b/ably/pubsub/http/auth.py @@ -55,7 +55,7 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, # RSA4: token auth unless use_token_auth says otherwise, or nothing but a key is given if options.use_token_auth is None: must_use_token_auth = bool(options.auth_callback or options.auth_url - or options.auth_token or options.token_details) + or options.token or options.token_details) else: must_use_token_auth = options.use_token_auth must_not_use_token_auth = options.use_token_auth is False @@ -83,8 +83,8 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, raise IncompatibleClientIdException( f"clientId '{options.client_id}' in the client options does not match the " f"token's clientId '{token_client_id}'", 400, 40102) - elif options.auth_token: - self.__token_details = TokenDetails(token=options.auth_token) + elif options.token: + self.__token_details = TokenDetails(token=options.token) else: self.__token_details = None @@ -94,7 +94,7 @@ def __init__(self, ably: DefaultPubSubHttpClient | DefaultPubSubRealtimeClient, log.debug("using token auth with auth_url") elif options.key_secret: log.debug("using token auth with client-side signing") - elif options.auth_token: + elif options.token: log.debug("using token auth with supplied token only") elif options.token_details: log.debug("using token auth with supplied token_details") diff --git a/ably/pubsub/http/http.py b/ably/pubsub/http/http.py index dd738c1a..a426a4e8 100644 --- a/ably/pubsub/http/http.py +++ b/ably/pubsub/http/http.py @@ -50,17 +50,10 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, - `client_id`: Undocumented - `endpoint`: Endpoint specifies either a routing policy name or fully qualified domain name to connect to Ably. - - `rest_host`: Deprecated: this property is deprecated and will - be removed in a future version. The host to connect to. - Defaults to rest.ably.io - - `environment`: Deprecated: this property is deprecated and - will be removed in a future version. The environment to use. - Defaults to 'production' - `port`: The port to connect to. Defaults to 80 - `tls_port`: The tls_port to connect to. Defaults to 443 - `tls`: Specifies whether the client should use TLS. Defaults to True - - `auth_token`: Undocumented - `auth_callback`: Undocumented - `auth_url`: Undocumented """ @@ -71,7 +64,7 @@ def __init__(self, key: Optional[str] = None, token: Optional[str] = None, "Provider either a key or key_name & key_secret", 401, 40106) # RSA4: a key given alongside a token is kept, to renew the token with if token is not None: - options = Options(key=key, auth_token=token, **kwargs) + options = Options(key=key, token=token, **kwargs) elif token_details is not None: if not isinstance(token_details, TokenDetails): raise InvalidArgumentException("token_details must be an instance of TokenDetails", 401, 40106) diff --git a/ably/pubsub/realtime/realtime.py b/ably/pubsub/realtime/realtime.py index 36b1118d..291c6331 100644 --- a/ably/pubsub/realtime/realtime.py +++ b/ably/pubsub/realtime/realtime.py @@ -55,13 +55,6 @@ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEve **kwargs: client options endpoint: str Endpoint specifies either a routing policy name or fully qualified domain name to connect to Ably. - realtime_host: str - Deprecated: this property is deprecated and will be removed in a future version. - Enables a non-default Ably host to be specified for realtime connections. - For development environments only. The default value is realtime.ably.io. - environment: str - Deprecated: this property is deprecated and will be removed in a future version. - Enables a custom environment to be used with the Ably service. Defaults to `production` realtime_request_timeout: float Timeout (in milliseconds) for the wait of acknowledgement for operations performed via a realtime connection. Operations include establishing a connection with Ably, or sending a HEARTBEAT, @@ -77,9 +70,6 @@ def __init__(self, key: Optional[str] = None, loop: Optional[asyncio.AbstractEve An array of fallback hosts to be used in the case of an error necessitating the use of an alternative host. If you have been provided a set of custom fallback hosts by Ably, please specify them here. - connection_state_ttl: float - The duration that Ably will persist the connection state for when a Realtime client is abruptly - disconnected. suspended_retry_timeout: float When the connection enters the SUSPENDED state, after this delay, if the state is still SUSPENDED, the client library attempts to reconnect automatically. The default is 30 seconds. diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index 55a1cf9f..d672ed38 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -115,8 +115,7 @@ def create_http_client(**kwargs) -> PubSubHttpClient: cannot be changed once the client has authenticated. endpoint: str Either a routing policy name or a fully qualified domain name to connect to. - Defaults to `main`. Incompatible with `environment`, `rest_host` and - `realtime_host`. + Defaults to `main`. tls: bool Whether the client should use TLS. Defaults to True. port: int @@ -155,13 +154,6 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Add a `request_id` query parameter to every request, which makes a request traceable in Ably's logs. Defaults to False. - **Deprecated** - - rest_host: str - Deprecated: this option will be removed in a future version. Use `endpoint`. - environment: str - Deprecated: this option will be removed in a future version. Use `endpoint`. - Returns ------- PubSubHttpClient @@ -170,9 +162,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Raises ------ AblyException - With code 40106, if no means of authenticating is given, if `key` is combined - with `key_name` or `key_secret`, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + With code 40106, if no means of authenticating is given, or if `key` is + combined with `key_name` or `key_secret`. It is also a ValueError. IncompatibleClientIdException If `client_id` conflicts with the client ID of `token_details`. """ @@ -239,11 +230,6 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: The decoder used to apply delta-encoded messages on channels configured with `params={"delta": "vcdiff"}`. Pass an :class:`AblyVCDiffDecoder`. - **Deprecated** - - realtime_host: str - Deprecated: this option will be removed in a future version. Use `endpoint`. - Returns ------- PubSubRealtimeClient @@ -252,9 +238,8 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: Raises ------ AblyException - With code 40106, if no means of authenticating is given, if `key` is combined - with `key_name` or `key_secret`, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + With code 40106, if no means of authenticating is given, or if `key` is + combined with `key_name` or `key_secret`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubRealtimeClient(**kwargs) diff --git a/ably/pubsub/server/sync.py b/ably/pubsub/server/sync.py index c5d8cbe1..75039039 100644 --- a/ably/pubsub/server/sync.py +++ b/ably/pubsub/server/sync.py @@ -54,9 +54,8 @@ def create_http_client(**kwargs) -> PubSubHttpClient: Raises ------ AblyException - With code 40106, if no means of authenticating is given, if `key` is combined - with `key_name` or `key_secret`, or if `endpoint` is combined with - `environment`, `rest_host` or `realtime_host`. It is also a ValueError. + With code 40106, if no means of authenticating is given, or if `key` is + combined with `key_name` or `key_secret`. It is also a ValueError. """ with factory_construction(): return _DefaultPubSubHttpClientSync(**kwargs) diff --git a/ably/pubsub/types/authoptions.py b/ably/pubsub/types/authoptions.py index 18f0fb30..36315c54 100644 --- a/ably/pubsub/types/authoptions.py +++ b/ably/pubsub/types/authoptions.py @@ -3,7 +3,7 @@ class AuthOptions: def __init__(self, auth_callback=None, auth_url=None, auth_method='GET', - auth_token=None, auth_headers=None, auth_params=None, + token=None, auth_headers=None, auth_params=None, key_name=None, key_secret=None, key=None, query_time=False, token_details=None, use_token_auth=None, default_token_params=None): @@ -18,7 +18,7 @@ def __init__(self, auth_callback=None, auth_url=None, auth_method='GET', self.auth_options['key_secret'] = key_secret self.set_key(key) - self.__auth_token = auth_token + self.__token = token self.__token_details = token_details self.__use_token_auth = use_token_auth default_token_params = default_token_params or {} @@ -97,13 +97,14 @@ def key_secret(self): def key_secret(self, value): self.auth_options['key_secret'] = value + # TO3j2 @property - def auth_token(self): - return self.__auth_token + def token(self): + return self.__token - @auth_token.setter - def auth_token(self, value): - self.__auth_token = value + @token.setter + def token(self, value): + self.__token = value @property def auth_headers(self): diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index e33bbb97..84551f46 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -4,7 +4,6 @@ from ably.pubsub.transport.defaults import Defaults from ably.pubsub.types.authoptions import AuthOptions -from ably.pubsub.util.exceptions import InvalidArgumentException log = logging.getLogger(__name__) @@ -26,9 +25,9 @@ def decode(self, delta: bytes, base: bytes) -> bytes: class Options(AuthOptions): - def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, port=0, + def __init__(self, client_id=None, tls=True, port=0, tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None, - environment=None, http_open_timeout=None, http_request_timeout=None, + http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, idempotent_rest_publishing=None, loop=None, auto_connect=True, @@ -39,15 +38,6 @@ def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, super().__init__(**kwargs) - # REC1b1: endpoint is incompatible with deprecated options - if endpoint is not None: - if environment is not None or rest_host is not None or realtime_host is not None: - raise InvalidArgumentException( - message='endpoint is incompatible with any of environment, rest_host or realtime_host', - status_code=400, - code=40106, - ) - # TODO check these defaults if fallback_retry_timeout is None: fallback_retry_timeout = Defaults.fallback_retry_timeout @@ -74,43 +64,15 @@ def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, if connectivity_check_url is None: connectivity_check_url = Defaults.connectivity_check_url - connection_state_ttl = Defaults.connection_state_ttl - if suspended_retry_timeout is None: suspended_retry_timeout = Defaults.suspended_retry_timeout - if environment is not None and rest_host is not None: - raise InvalidArgumentException( - message='specify rest_host or environment, not both', - status_code=400, - code=40106, - ) - - if environment is not None and realtime_host is not None: - raise InvalidArgumentException( - message='specify realtime_host or environment, not both', - status_code=400, - code=40106, - ) - if idempotent_rest_publishing is None: from ably.pubsub.version import api_version idempotent_rest_publishing = api_version >= '1.2' - if environment is not None and endpoint is None: - log.warning("environment client option is deprecated, please use endpoint instead") - endpoint = environment - - # REC1d: restHost or realtimeHost option - # REC1d1: restHost takes precedence over realtimeHost - if rest_host is not None and endpoint is None: - log.warning("rest_host client option is deprecated, please use endpoint instead") - endpoint = rest_host - elif realtime_host is not None and endpoint is None: - # REC1d2: realtimeHost if restHost not specified - log.warning("realtime_host client option is deprecated, please use endpoint instead") - endpoint = realtime_host - + # REC1a: the default routing policy. ably-python omits the deprecated environment, + # rest_host and realtime_host options (TO3k1-3), so REC1c and REC1d do not apply if endpoint is None: endpoint = Defaults.endpoint @@ -137,7 +99,6 @@ def __init__(self, client_id=None, tls=True, rest_host=None, realtime_host=None, self.__idempotent_rest_publishing = idempotent_rest_publishing self.__loop = loop self.__auto_connect = auto_connect - self.__connection_state_ttl = connection_state_ttl self.__suspended_retry_timeout = suspended_retry_timeout self.__connectivity_check_url = connectivity_check_url self.__add_request_ids = add_request_ids @@ -276,14 +237,6 @@ def loop(self): def auto_connect(self): return self.__auto_connect - @property - def connection_state_ttl(self): - return self.__connection_state_ttl - - @connection_state_ttl.setter - def connection_state_ttl(self, value): - self.__connection_state_ttl = value - @property def suspended_retry_timeout(self): return self.__suspended_retry_timeout diff --git a/test/ably/http/httpinit_test.py b/test/ably/http/httpinit_test.py index b8202a3b..702cafac 100644 --- a/test/ably/http/httpinit_test.py +++ b/test/ably/http/httpinit_test.py @@ -28,7 +28,7 @@ def per_protocol_setup(self, use_binary_protocol): @dont_vary_protocol def test_with_token(self): ably = create_http_client(token="foo") - assert ably.options.auth_token == "foo", "Token not set at options" + assert ably.options.token == "foo", "Token not set at options" @dont_vary_protocol def test_with_token_details(self): @@ -68,7 +68,7 @@ def test_with_options_auth_url(self): # RSC11 @dont_vary_protocol - def test_rest_host_and_environment(self): + def test_endpoint(self): # endpoint host ably = create_http_client(token='foo', endpoint="some.other.host") assert "some.other.host" == ably.options.get_host(), "Unexpected host mismatch" @@ -83,10 +83,10 @@ def test_rest_host_and_environment(self): host = ably.options.get_host() assert "sandbox.realtime.ably-nonprod.net" == host, f"Unexpected host mismatch {host}" - # both, as per #TO3k2 - with pytest.raises(AblyException): - ably = create_http_client(token='foo', rest_host="some.other.host", - endpoint="some.other.environment") + # The deprecated host options are omitted (TO3k1-3) + for option in ('environment', 'rest_host', 'realtime_host'): + with pytest.raises(TypeError): + create_http_client(token='foo', **{option: 'some.other.host'}) # RSC15 @dont_vary_protocol diff --git a/test/unit/options_test.py b/test/unit/options_test.py index b58e4ce1..fb68159f 100644 --- a/test/unit/options_test.py +++ b/test/unit/options_test.py @@ -5,22 +5,13 @@ from ably.pubsub.util.exceptions import AblyException -# REC1b1: endpoint is incompatible with deprecated options -def test_options_should_fail_early_with_incompatible_client_options(): - # REC1b1: endpoint with environment - with pytest.raises(AblyException) as exinfo: - Options(endpoint="foo", environment="foo") - assert exinfo.value.code == 40106 - - # REC1b1: endpoint with rest_host - with pytest.raises(AblyException) as exinfo: - Options(endpoint="foo", rest_host="foo") - assert exinfo.value.code == 40106 - - # REC1b1: endpoint with realtime_host - with pytest.raises(AblyException) as exinfo: - Options(endpoint="foo", realtime_host="foo") - assert exinfo.value.code == 40106 +# TO3k1-3: the deprecated environment, rest_host and realtime_host options are omitted, +# and with them the duplicate auth_token and the unused connection_state_ttl +@pytest.mark.parametrize('option', ['environment', 'rest_host', 'realtime_host', 'auth_token', + 'connection_state_ttl']) +def test_options_should_reject_removed_options(option): + with pytest.raises(TypeError): + Options(**{option: 'foo'}) # REC1a @@ -71,56 +62,6 @@ def test_options_should_return_localhost(): assert not opts.get_fallback_hosts() -# REC1c1: environment with rest_host or realtime_host is invalid -def test_options_should_fail_with_environment_and_rest_or_realtime_host(): - # REC1c1: environment with rest_host - with pytest.raises(AblyException) as exinfo: - Options(environment="foo", rest_host="bar") - assert exinfo.value.code == 40106 - - # REC1c1: environment with realtime_host - with pytest.raises(AblyException) as exinfo: - Options(environment="foo", realtime_host="bar") - assert exinfo.value.code == 40106 - - -# REC1c2: environment defines production routing policy ID -def test_options_with_environment_should_return_routing_policy_hostnames(): - opts = Options(environment="foo") - # REC1c2: primary domain is [id].realtime.ably.net - assert opts.get_host() == "foo.realtime.ably.net" - # REC2c5: fallback domains for production routing policy ID via environment - assert "foo.a.fallback.ably-realtime.com" in opts.get_fallback_hosts() - assert "foo.e.fallback.ably-realtime.com" in opts.get_fallback_hosts() - - -# REC1d1: rest_host takes precedence for primary domain -def test_options_with_rest_host_should_return_rest_host(): - opts = Options(rest_host="custom.example.com") - # REC1d1: primary domain is the value of the restHost option - assert opts.get_host() == "custom.example.com" - # REC2c6: fallback domains for restHost is empty - assert not opts.get_fallback_hosts() - - -# REC1d2: realtime_host if rest_host not specified -def test_options_with_realtime_host_should_return_realtime_host(): - opts = Options(realtime_host="custom.example.com") - # REC1d2: primary domain is the value of the realtimeHost option - assert opts.get_host() == "custom.example.com" - # REC2c6: fallback domains for realtimeHost is empty - assert not opts.get_fallback_hosts() - - -# REC1d1: rest_host takes precedence over realtime_host -def test_options_with_rest_host_takes_precedence_over_realtime_host(): - opts = Options(rest_host="rest.example.com", realtime_host="realtime.example.com") - # REC1d1: restHost takes precedence - assert opts.get_host() == "rest.example.com" - # REC2c6: fallback domains is empty - assert not opts.get_fallback_hosts() - - # REC2a2: fallback_hosts value is used when specified def test_options_with_fallback_hosts_should_use_specified_hosts(): custom_fallbacks = ["fallback1.example.com", "fallback2.example.com"] @@ -206,7 +147,6 @@ def test_options_localhost_no_fallback_hosts(): ({'key': 'appId.keyId:keySecret', 'key_name': 'appId.keyId'}, 401), ({'token_details': 'not-token-details'}, 401), ({'token': 'a-token', 'use_token_auth': False}, 401), - ({'key': 'appId.keyId:keySecret', 'endpoint': 'foo', 'environment': 'foo'}, 400), ]) def test_invalid_client_arguments_raise_40106(kwargs, status_code): with pytest.raises(AblyException) as exinfo: diff --git a/test/unit/websockettransport_test.py b/test/unit/websockettransport_test.py index 3cf1fffb..1c0e3015 100644 --- a/test/unit/websockettransport_test.py +++ b/test/unit/websockettransport_test.py @@ -6,7 +6,7 @@ def _connect_url(host='example.com', **option_kwargs): connection_manager = MagicMock() - connection_manager.options = Options(auth_token='foo', **option_kwargs) + connection_manager.options = Options(token='foo', **option_kwargs) transport = WebSocketTransport(connection_manager, host, {'format': 'json'}) with patch.object(transport, 'ws_connect', MagicMock()) as mock_ws_connect: with patch('ably.pubsub.transport.websockettransport.asyncio.create_task') as mock_create_task: diff --git a/test/uts/deviations.md b/test/uts/deviations.md index a607c6d3..22eeebc1 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,9 +33,9 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1304 Test IDs -(1313 tests, 1438 cases) pass, 153 (153 tests, 155 cases) are gated behind -`RUN_DEVIATIONS`, and 15 (15 tests, 15 cases) cannot be run at all. The three groups are +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs +(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind +`RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, while their passing parts count among the passing tests and cases. Every gated test has @@ -2003,8 +2003,9 @@ through their setups. `ConnectionDetails.from_dict` parses `connectionStateTtl` (`types/connectiondetails.py:19`) and nothing ever reads it. `ConnectionManager.start_suspend_timer` (`connectionmanager.py:745`) uses `Defaults.connection_state_ttl` — 120000 — directly, and -no client option overrides it (`types/options.py:64` accepts the keyword and then discards -it). A server that shortens or lengthens the TTL is ignored. +no client option overrides it (TO3 defines none, and the `Options.connection_state_ttl` +property that copied the default and was read by nothing was removed in 4.0.0). A server +that shortens or lengthens the TTL is ignored. **Tests affected:** every test that has to reach a SUSPENDED connection — nine across the connection, channel and presence suites — sends the specification's `connectionStateTtl` and @@ -2207,7 +2208,6 @@ pseudocode is not mistaken for non-compliance. | RSL2, RSL11b, RSL15b | `:` is `%3A` | Left literal, from `safe=':'`. RFC 3986 allows `:` in a path segment and Ably uses it for namespaces, so the server receives the same value | Intentional | | RTC12 / RSC1, RSC1a, RSC1c | A string constructor argument is an API key when it contains `:` and a token when it does not | `AblyRest.__init__` treats its first positional argument as a key unconditionally and hands it to `AuthOptions.set_key`, which requires exactly two colon-separated parts, so a token string raises 40101/401 "key of not len 2 parameters". A token is supplied through the separate `token` or `token_details` arguments. The empty-string case is compliant | Intentional / SDK-wide: the constructor takes credentials as distinct named arguments and has no string-sniffing path to restore | | RSC18 | The constructor rejects basic auth over HTTP | Construction succeeds; 40103 is raised from `make_request` when a request needing Basic Auth is attempted, and no request goes out. RSA1/RSC18 say only "any attempt to use" | Compliant; the UTS is stricter than its source | -| REC1b1, REC1c1 | Code 40000, or a message containing "invalid" or "conflict" | 400/40106 with a specific message. The features spec mandates no code | Cosmetic | | RSAN1a3 | Code 40003 for a missing `Annotation.type` | 400/40000 | Cosmetic; worth aligning cross-SDK | | RSH1a | Empty `recipient` or `data` rejected with code 40000, the error reaching the caller from the server | `PushAdmin.publish` validates its arguments itself and raises before touching the HTTP layer: an empty or non-dict `recipient` or `data` raises an `AblyException` with 400/40000, the code the server gives, which is also a `ValueError`. So no request is made and there is no server error; the integration tier's `test_rsh1a_push_publish_invalid_recipient` asserts the local error's code | Compliant: RSH1a asks for empty values to be "immediately rejected". The specification's test would need a recipient the SDK will send and the server will reject, an unknown `transportType` say, to exercise the server-side path it describes | | HP6 | `errorCode` is a number | The raw header string, `'40101'` | Open bug, trivial | @@ -2273,7 +2273,7 @@ largest, the typed views, has its own section at the end of this file. ## Mock Infrastructure Limitations Tests that cannot be implemented as written, kept as skipped stubs carrying their Test -IDs so the specification's coverage is still accounted for. Fifteen in total, none of +IDs so the specification's coverage is still accounted for. Twenty-seven in total, none of them from `uts/objects`. Two of the entries are caused by the SDK rather than by the mock, but they land here because the effect is the same: no test can observe the behaviour. @@ -2351,6 +2351,24 @@ Optional per TO3k7, and `REC1b1` and `REC2a1` scope their checks to libraries th support it, so these are skipped as not applicable rather than recorded as deviations. +### `environment`, `restHost` and `realtimeHost` are omitted — 12 tests + +**Spec points:** REC1b1 (`endpoint-conflicts-environment-0`, `-resthost-1`, +`-realtimehost-2`), REC1c1 (two), REC1c2, REC1d, REC1d1, REC1d2, REC2c5, and REC2c6 (two), +all in `fallback_test.py`. + +TO3k1, TO3k2 and TO3k3 mark the three options optional and deprecated, and say a library +with no backwards-compatibility requirement may omit them. ably-python 4.0.0 is a breaking +release and omits them, so `endpoint` is the only way to name a host, passing any of the +three raises `TypeError`, and the REC1 and REC2 clauses that only they select do not apply. +The tests are skipped as not applicable, as the `fallbackHostsUseDefault` ones are. + +Four tests elsewhere use one of the three only to name a host, and pass with `endpoint` in +its place, which REC1b2 resolves to the same host: `TG/pagination-relative-urls-5`, +`TO3/client-options-custom-hosts-1` and `RTN17g/empty-fallback-set-error-0` each carry a +NOTE saying so, and `TO/conflicting-options-validation-1` asserts the `TypeError` for its +restHost-and-endpoint case, there being no conflict left to construct. + ## Investigated and not defects Claims raised during derivation, investigated, and found not to be SDK faults. They are @@ -3517,8 +3535,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 155 failures, 1586 passes and 15 skips with the variable -set, and 170 skips and 1586 passes without it, the 1586 being 1438 derived cases and 148 +it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable +set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/connection/fallback_hosts_test.py b/test/uts/realtime/unit/connection/fallback_hosts_test.py index 69294ba8..b24dfb68 100644 --- a/test/uts/realtime/unit/connection/fallback_hosts_test.py +++ b/test/uts/realtime/unit/connection/fallback_hosts_test.py @@ -208,9 +208,10 @@ def on_connection_attempt(conn): conn.respond_with_refused() mock_ws = MockWebSocket(on_connection_attempt=on_connection_attempt) - # REC2c2: a custom domain has no fallback set of its own + # REC2c2: a custom domain has no fallback set of its own. The spec sets realtimeHost, + # which ably-python omits (TO3k3); endpoint names the same host client = fallback_client( - mock_ws, realtime_host='custom.example.com', realtime_request_timeout=300, + mock_ws, endpoint='custom.example.com', realtime_request_timeout=300, disconnected_retry_timeout=60000) client.connect() diff --git a/test/uts/rest/unit/fallback_test.py b/test/uts/rest/unit/fallback_test.py index d5f77a38..477d79f0 100644 --- a/test/uts/rest/unit/fallback_test.py +++ b/test/uts/rest/unit/fallback_test.py @@ -43,6 +43,13 @@ def connect_successfully(conn): conn.respond_with_success() +# TO3k1, TO3k2 and TO3k3 let a library with no backwards-compatibility requirement omit +# the deprecated environment, restHost and realtimeHost options, and ably-python 4 does. +OMITTED_HOST_OPTIONS_SKIP = ( + 'environment, restHost and realtimeHost are optional and deprecated (features.md TO3k1-3), ' + 'and ably-python omits them, so the REC1 and REC2 behaviour they select does not apply.') + + def new_mock(): return MockHttpClient(on_connection_attempt=connect_successfully) @@ -529,6 +536,7 @@ async def test_rec1b4_production_routing_policy(): # UTS: rest/unit/REC1b1/endpoint-conflicts-environment-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1b1_endpoint_conflicts_environment(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), endpoint='test', environment='production') @@ -541,6 +549,7 @@ async def test_rec1b1_endpoint_conflicts_environment(): # UTS: rest/unit/REC1b1/endpoint-conflicts-resthost-1 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1b1_endpoint_conflicts_resthost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), endpoint='test', rest_host='custom.host.com') @@ -552,6 +561,7 @@ async def test_rec1b1_endpoint_conflicts_resthost(): # UTS: rest/unit/REC1b1/endpoint-conflicts-realtimehost-2 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1b1_endpoint_conflicts_realtimehost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), endpoint='test', realtime_host='custom.realtime.com') @@ -572,6 +582,7 @@ async def test_rec1b1_endpoint_conflicts_fallback_default(): # UTS: rest/unit/REC1c2/environment-sets-primary-domain-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1c2_environment_sets_primary_domain(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -584,6 +595,7 @@ async def test_rec1c2_environment_sets_primary_domain(): # UTS: rest/unit/REC1c1/environment-conflicts-resthost-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1c1_environment_conflicts_resthost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), environment='sandbox', rest_host='custom.host.com') @@ -595,6 +607,7 @@ async def test_rec1c1_environment_conflicts_resthost(): # UTS: rest/unit/REC1c1/environment-conflicts-realtimehost-1 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1c1_environment_conflicts_realtimehost(): with pytest.raises(AblyException) as excinfo: rest_client(new_mock(), environment='sandbox', realtime_host='custom.realtime.com') @@ -606,6 +619,7 @@ async def test_rec1c1_environment_conflicts_realtimehost(): # UTS: rest/unit/REC1d1/resthost-sets-primary-domain-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1d1_resthost_sets_primary_domain(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -618,6 +632,7 @@ async def test_rec1d1_resthost_sets_primary_domain(): # UTS: rest/unit/REC1d2/realtimehost-sets-primary-domain-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1d2_realtimehost_sets_primary_domain(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -630,6 +645,7 @@ async def test_rec1d2_realtimehost_sets_primary_domain(): # UTS: rest/unit/REC1d/resthost-precedence-over-realtimehost-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec1d_resthost_precedence_over_realtimehost(): mock_http = new_mock() mock_http.queue_response(200, TIME_BODY) @@ -750,6 +766,7 @@ async def test_rec2c4_production_endpoint_fallback_domains(): # UTS: rest/unit/REC2c5/production-environment-fallback-domains-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec2c5_production_environment_fallback_domains(): mock_http = new_mock() mock_http.queue_response(500, error_body(500)) @@ -765,6 +782,7 @@ async def test_rec2c5_production_environment_fallback_domains(): # UTS: rest/unit/REC2c6/custom-resthost-no-fallbacks-0 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec2c6_custom_resthost_no_fallbacks(): mock_http = new_mock() mock_http.queue_response(500, error_body(500)) @@ -780,6 +798,7 @@ async def test_rec2c6_custom_resthost_no_fallbacks(): # UTS: rest/unit/REC2c6/custom-realtimehost-no-fallbacks-1 +@pytest.mark.skip(reason=OMITTED_HOST_OPTIONS_SKIP) async def test_rec2c6_custom_realtimehost_no_fallbacks(): mock_http = new_mock() mock_http.queue_response(500, error_body(500)) diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index 471b940a..a0a46c33 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -9,7 +9,6 @@ from ably.pubsub.types.authoptions import AuthOptions from ably.pubsub.types.options import Options from ably.pubsub.types.tokendetails import TokenDetails -from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client from test.uts.helpers.deviations import deviation from test.uts.helpers.mock_http import MockHttpClient @@ -86,11 +85,12 @@ async def test_to3_client_options_attributes(): def test_to3_client_options_custom_hosts(): options = Options( key='appId.keyId:keySecret', - rest_host='custom.ably.example.com', + # NOTE: the spec sets restHost, which ably-python omits (TO3k2); endpoint names the + # same host + endpoint='custom.ably.example.com', fallback_hosts=['fallback1.example.com', 'fallback2.example.com'], ) - # rest_host is deprecated in favour of endpoint (REC1d), which is where it is held assert options.endpoint == 'custom.ably.example.com' # The fallbacks are kept as given, in the random order RSC15a tries them in assert sorted(options.fallback_hosts) == ['fallback1.example.com', 'fallback2.example.com'] @@ -184,9 +184,10 @@ async def auth_callback(params): assert client.auth.auth_options.auth_callback is auth_callback # Case 2: restHost + endpoint conflict - with pytest.raises(AblyException) as excinfo: + # NOTE: ably-python omits restHost (TO3k2), so there is no conflict to construct; the + # option itself is rejected + with pytest.raises(TypeError): Options(key='appId.keyId:keySecret', rest_host='custom.host.com', endpoint='test') - assert 'rest_host' in excinfo.value.message or 'endpoint' in excinfo.value.message # Case 3: no auth options # The missing-credentials check is a constructor argument check, so it raises ValueError diff --git a/test/uts/rest/unit/types/paginated_result_test.py b/test/uts/rest/unit/types/paginated_result_test.py index 22f0f1bb..15661d3a 100644 --- a/test/uts/rest/unit/types/paginated_result_test.py +++ b/test/uts/rest/unit/types/paginated_result_test.py @@ -265,7 +265,8 @@ async def test_tg_pagination_relative_urls(): ([{'id': 'item2'}], {}), ]), ) - client = rest_client(mock_http, rest_host='rest.ably.io') + # NOTE: the spec sets restHost, which ably-python omits (TO3k2); endpoint names the same host + client = rest_client(mock_http, endpoint='rest.ably.io') channel = client.channels.get('test') page1 = await channel.history() From 1a97b10d29fb12fb49cc48c2121d9b153bc1a0b2 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:37:59 +0000 Subject: [PATCH 7/8] pubsub: check publishes against a configurable max_message_size TO3l8 makes maxMessageSize a client option, default 65536, and RSL1i has a publish whose messages exceed it between them rejected with 40009 before it is sent. ably-python had no such option, and REST publishes were never checked. The realtime publish read the limit with getattr(options, 'max_message_size', 65536), which always fell back to the default, and measured the whole encoded message, msgpack or JSON with ASCII escaping, rather than the TM6 size, so it counted the id, the encoding and every other field the specification leaves out. max_message_size is now a client option, defaulting to Defaults.max_message_size. message_size measures a message per TM6: the lengths of its name, clientId and string data, the bytes of binary or encrypted data, and the length of the JSON of object data and extras. The REST and realtime publishes both check their messages with it, after encryption and before encoding. CD2c, the connectionDetails override for realtime, is not implemented, since ably-python does not parse that field. The RSL1i UTS test is un-gated. Its at-limit case is five bytes over once TM6 counts the name, so the derived test corrects the fixture, which test/uts/deviations.md records as a spec error along with the updated counts. This is a breaking change: a REST publish over 64 KiB now raises an AblyException with code 40009 instead of being sent, and some realtime publishes that were refused are now sent. Callers whose account allows larger messages can raise max_message_size. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 18 +++++++++ ably/pubsub/http/channel.py | 4 ++ ably/pubsub/realtime/channel.py | 7 ++-- ably/pubsub/server/__init__.py | 4 ++ ably/pubsub/transport/defaults.py | 2 + ably/pubsub/types/message.py | 36 ++++++++++++++++++ ably/pubsub/types/options.py | 11 +++++- ably/pubsub/util/helper.py | 29 -------------- test/unit/message_test.py | 38 +++++++++++++++++++ test/uts/deviations.md | 26 +++++++------ test/uts/rest/unit/channel/publish_test.py | 8 ++-- .../uts/rest/unit/types/options_types_test.py | 6 +-- 12 files changed, 136 insertions(+), 53 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 9b609914..801db6dc 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -516,6 +516,24 @@ pubsub_realtime_client = create_realtime_client(key='xxx', endpoint='custom.ably pubsub_http_client = create_http_client(token='xxx') ``` +### REST publishes are checked against `max_message_size` + +`max_message_size` is now a client option, defaulting to 65536 bytes (64 KiB). A publish +whose messages exceed it between them raises an `AblyException` with code 40009 and is not +sent. In 3.x the realtime client checked a fixed 64 KiB limit and REST publishes were not +checked at all, so an oversized REST publish now fails before it reaches Ably rather than +being rejected by the server. + +Both clients now measure a message the way the Ably specification does: the length of its +name, client ID and string data, the byte length of binary data, and the length of the JSON +of object data and of extras. 3.x measured the realtime publish as the whole encoded +message, so some realtime publishes that 3.x refused are now sent. Raise the limit only if +your Ably account allows larger messages: + +```python +pubsub_http_client = create_http_client(key='xxx', max_message_size=131072) +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/http/channel.py b/ably/pubsub/http/channel.py index e2bd13a0..134de023 100644 --- a/ably/pubsub/http/channel.py +++ b/ably/pubsub/http/channel.py @@ -16,6 +16,7 @@ assign_idempotent_ids, make_message_response_handler, make_single_message_response_handler, + validate_message_size, ) from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.types.presence import Presence @@ -76,6 +77,9 @@ def __publish_request_body(self, messages): request_body_list.append(m) + # RSL1i + validate_message_size(request_body_list, self.ably.options.max_message_size) + request_body = [ message.as_dict(binary=self.ably.options.use_binary_protocol) for message in request_body_list] diff --git a/ably/pubsub/realtime/channel.py b/ably/pubsub/realtime/channel.py index 7da4ed1f..c4421fbe 100644 --- a/ably/pubsub/realtime/channel.py +++ b/ably/pubsub/realtime/channel.py @@ -17,14 +17,14 @@ from ably.pubsub.types.channeloptions import ChannelOptions from ably.pubsub.types.channelstate import ChannelState, ChannelStateChange from ably.pubsub.types.flags import Flag, has_flag -from ably.pubsub.types.message import Message, MessageAction, MessageVersion +from ably.pubsub.types.message import Message, MessageAction, MessageVersion, validate_message_size from ably.pubsub.types.mixins import DecodingContext from ably.pubsub.types.operations import MessageOperation, PublishResult, UpdateDeleteResult from ably.pubsub.types.presence import PresenceMessage from ably.pubsub.util.clock import select_clock from ably.pubsub.util.eventemitter import EventEmitter from ably.pubsub.util.exceptions import AblyException, IncompatibleClientIdException -from ably.pubsub.util.helper import Timer, is_callable_or_coroutine, validate_message_size +from ably.pubsub.util.helper import Timer, is_callable_or_coroutine if TYPE_CHECKING: from ably.pubsub.realtime.realtime import DefaultPubSubRealtimeClient @@ -464,8 +464,7 @@ async def publish(self, *args, **kwargs) -> PublishResult: encoded_messages.append(msg_dict) # RSL1i: Check message size limit - max_message_size = getattr(self.ably.options, 'max_message_size', 65536) # 64KB default - validate_message_size(encoded_messages, self.ably.options.use_binary_protocol, max_message_size) + validate_message_size(messages, self.ably.options.max_message_size) # RTL6c: Check connection and channel state self._throw_if_unpublishable_state() diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index d672ed38..d203ae8d 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -153,6 +153,10 @@ def create_http_client(**kwargs) -> PubSubHttpClient: add_request_ids: bool Add a `request_id` query parameter to every request, which makes a request traceable in Ably's logs. Defaults to False. + max_message_size: int + The largest total size of the messages one publish may carry, counting each + message's name, data, client ID and extras. A larger publish raises an + AblyException with code 40009 and is not sent. The default is 65536 (64 KiB). Returns ------- diff --git a/ably/pubsub/transport/defaults.py b/ably/pubsub/transport/defaults.py index d239656a..cdb93d73 100644 --- a/ably/pubsub/transport/defaults.py +++ b/ably/pubsub/transport/defaults.py @@ -27,6 +27,8 @@ class Defaults: fallback_retry_timeout = 600000 # 10min + max_message_size = 65536 # 64KiB + @staticmethod def get_port(options): if options.tls: diff --git a/ably/pubsub/types/message.py b/ably/pubsub/types/message.py index ad574154..aaf1b4d6 100644 --- a/ably/pubsub/types/message.py +++ b/ably/pubsub/types/message.py @@ -1,4 +1,5 @@ import base64 +import json import logging import os from enum import IntEnum @@ -393,6 +394,41 @@ def update_inner_message_fields(proto_msg: dict): msg_index = msg_index + 1 +def message_size(message): + """The size of a message as TM6 defines it, for checking against maxMessageSize (TO3l8).""" + def json_length(value): + return len(json.dumps(value, separators=(',', ':'), ensure_ascii=False)) + + # TM6a: the sum of the name, data, clientId and extras sizes; TM6e: an absent one is zero + size = len(message.name or '') + len(message.client_id or '') + if message.extras is not None: + # TM6d + size += json_length(message.extras) + + data = message.data + if isinstance(data, CipherData): + data = data.buffer + if isinstance(data, (bytes, bytearray, str)): + # TM6c: the bytes themselves, whichever protocol carries them; TM6f: a string's length + size += len(data) + elif isinstance(data, (dict, list)): + # TM6b + size += json_length(data) + return size + + +def validate_message_size(messages, max_message_size): + """Raises 40009 if the messages exceed max_message_size between them (RSL1i, TO3l8f).""" + size = sum(message_size(message) for message in messages) + if size > max_message_size: + raise AblyException( + f"Maximum size of messages that can be published at once exceeded " + f"(was {size} bytes; limit is {max_message_size} bytes)", + 400, + 40009, + ) + + def assign_idempotent_ids(messages): """Gives every message a library-generated id, unless one of them already has an id (RSL1k1).""" if all(message.id is None for message in messages): diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 84551f46..36597162 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -34,7 +34,7 @@ def __init__(self, client_id=None, tls=True, port=0, suspended_retry_timeout=None, connectivity_check_url=None, channel_retry_timeout=Defaults.channel_retry_timeout, add_request_ids=False, vcdiff_decoder: VCDiffDecoder = None, transport_params=None, echo_messages=True, - _test_options=None, **kwargs): + max_message_size=None, _test_options=None, **kwargs): super().__init__(**kwargs) @@ -67,6 +67,10 @@ def __init__(self, client_id=None, tls=True, port=0, if suspended_retry_timeout is None: suspended_retry_timeout = Defaults.suspended_retry_timeout + # TO3l8 + if max_message_size is None: + max_message_size = Defaults.max_message_size + if idempotent_rest_publishing is None: from ably.pubsub.version import api_version idempotent_rest_publishing = api_version >= '1.2' @@ -104,6 +108,7 @@ def __init__(self, client_id=None, tls=True, port=0, self.__add_request_ids = add_request_ids self.__vcdiff_decoder = vcdiff_decoder self.__transport_params = transport_params or {} + self.__max_message_size = max_message_size self.__test_options = _test_options self.__hosts = self.__get_hosts() @@ -271,6 +276,10 @@ def vcdiff_decoder(self): def transport_params(self): return self.__transport_params + @property + def max_message_size(self): + return self.__max_message_size + @property def _test_options(self): return self.__test_options diff --git a/ably/pubsub/util/helper.py b/ably/pubsub/util/helper.py index 934a1af1..f3bcf351 100644 --- a/ably/pubsub/util/helper.py +++ b/ably/pubsub/util/helper.py @@ -1,16 +1,11 @@ import asyncio import inspect -import json import random import string import time from typing import Callable, Dict, Tuple from urllib.parse import parse_qs, urlparse -import msgpack - -from ably.pubsub.util.exceptions import AblyException - def get_random_id(): # get random string of letters and digits @@ -76,30 +71,6 @@ def cancel(self): self._task.cancel() -def validate_message_size(encoded_messages: list, use_binary_protocol: bool, max_message_size: int) -> None: - """Validate that encoded messages don't exceed the maximum size limit. - - Args: - encoded_messages: List of encoded message dictionaries - use_binary_protocol: Whether to use binary (msgpack) or JSON encoding - max_message_size: Maximum allowed size in bytes - - Raises: - AblyException: If the encoded messages exceed the maximum size - """ - if use_binary_protocol: - size = len(msgpack.packb(encoded_messages, use_bin_type=True)) - else: - size = len(json.dumps(encoded_messages, separators=(',', ':')).encode('utf-8')) - - if size > max_message_size: - raise AblyException( - f"Maximum size of messages that can be published at once exceeded " - f"(was {size} bytes; limit is {max_message_size} bytes)", - 400, - 40009, - ) - def to_text(value): if value is None: return value diff --git a/test/unit/message_test.py b/test/unit/message_test.py index f49e9bf9..22e0b53a 100644 --- a/test/unit/message_test.py +++ b/test/unit/message_test.py @@ -1,4 +1,8 @@ +import pytest + import ably.pubsub.types.message +from ably.pubsub.types.message import Message, message_size, validate_message_size +from ably.pubsub.util.exceptions import AblyException # TM2a, TM2c, TM2f @@ -57,3 +61,37 @@ def test_invalid_base64_data_is_delivered_with_remaining_encoding(): assert message.data == 'encrypted-data-here' assert message.encoding == 'custom-encryption/base64' + + +# TM6a, TM6e +def test_message_size_sums_name_data_client_id_and_extras(): + assert message_size(Message()) == 0 + assert message_size(Message(name='event', data='hello', client_id='me')) == 12 + # TM6d: extras count as their JSON + assert message_size(Message(extras={'push': {'a': 1}})) == len('{"push":{"a":1}}') + # An id is not counted + assert message_size(Message(id='abc:0', data='x')) == 1 + + +# TM6b, TM6c, TM6f +def test_message_size_of_each_kind_of_data(): + # TM6f: a string is its length, and a non-ASCII one is not counted as its JSON escape + assert message_size(Message(data='h\u00e9llo')) == 5 + # TM6c: binary is its byte length, not the length of its base64 + assert message_size(Message(data=b'\x00' * 10)) == 10 + assert message_size(Message(data=bytearray(10))) == 10 + # TM6b: an object or array is the length of its JSON + assert message_size(Message(data={'k': 'v'})) == len('{"k":"v"}') + assert message_size(Message(data=[1, '\u00e9'])) == len('[1,"\u00e9"]') + + +# RSL1i, TO3l8f: the limit applies to the messages published together +def test_validate_message_size_raises_40009_above_the_limit(): + messages = [Message(name='a', data='x' * 4), Message(name='b', data='x' * 4)] + + validate_message_size(messages, 10) + with pytest.raises(AblyException) as excinfo: + validate_message_size(messages, 9) + + assert excinfo.value.code == 40009 + assert excinfo.value.status_code == 400 diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 22eeebc1..8ef1f73e 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs -(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1293 Test IDs +(1302 tests, 1427 cases) pass, 152 (152 tests, 154 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,17 +45,17 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 70 are REST and 83 realtime, which is 71 REST cases +within each. Of the gated Test IDs 69 are REST and 83 realtime, which is 70 REST cases and 84 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to -**61 distinct root causes**, 21 on the REST side and 40 on the realtime side. Two further +The 141 gated Test IDs that record SDK non-compliance — 141 tests, 143 cases — reduce to +**60 distinct root causes**, 20 on the REST side and 40 on the realtime side. Two further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a) or has nothing to assert against (the timezone split on -synthesized LEAVE timestamps), so the file carries **63 SDK root causes** in all. The remaining 11 gated Test IDs are +synthesized LEAVE timestamps), so the file carries **62 SDK root causes** in all. The remaining 11 gated Test IDs are specification faults, and reduce to 8. The objects tier records no SDK non-compliance; its specification faults are all corrected or worked around in tests that pass, and reduce to the seven candidate issues O.1–O.7, none of them filed. @@ -252,6 +252,11 @@ rather than merging." `RSA10k`'s setup depends on the same premise and cannot re - `RSL4/encoding-fixtures-ably-common-0` loads `encoding.json` from ably-common. No such file exists; RSL6a1 names `messages-encoding.json`, which has a different schema and runs in the decode direction. +- `RSL1i/message-size-limit-0` publishes `name: "event"` with `"x" * size` as the data + and expects the 1024-byte case to pass a 1024-byte limit, but TM6a counts the name as + well as the data, so that message is 1029 bytes. The derived test shortens the data by + the name's five bytes, so each message is the size its case states, and passes. Not + filed. ### Presence actions are written as strings @@ -1058,8 +1063,7 @@ on the `recover` parameter the connection never sends. | TB3 | `ChannelOptions.withCipherKey`. The nearest equivalent, `ably.util.crypto.get_default_params({'key': key})`, is not reachable from `ChannelOptions` | 1 | | RTL2i, TH6 | `hasBacklog` on `ChannelStateChange`. `Flag.HAS_BACKLOG` is defined (`types/flags.py:7`) but `_on_message` reads only RESUMED and HAS_PRESENCE (`channel.py:715-721`). See the UTS Spec Error above: the features spec makes this optional | 1 | | RTP6b | Subscribing to an *array* of presence actions. The list reaches `EventEmitter.on` and pyee uses the event as a dict key, so `presence.subscribe([ENTER, LEAVE], listener)` raises `TypeError: unhashable type: 'list'`. A fix has to fan the list out into one registration per action, and `unsubscribe` with it. `realtime/integration/presence_lifecycle_test.py::test_rtp4_bulk_enter_observed` adapts instead of gating: it registers the one listener once for `'enter'` and once for `'present'`, which is what the array form means, and the counted set is identical | 1 | -| RSL1i | REST publish never calls `validate_message_size`. The helper exists and is correct, but only `ably/realtime/channel.py:423` calls it, so an oversized REST publish goes out | 1 | -| TP5 | `size` on `PresenceMessage`. The related `maxMessageSize` gap is adapted rather than gated, below; `features.md` TM6 has no UTS test | 1 | +| TP5 | `size` on `PresenceMessage`. Published messages are measured per TM6 and checked against `max_message_size` (RSL1i, TO3l8), but presence messages are measured by nothing, and neither type exposes its size | 1 | ### Connection @@ -2218,7 +2222,7 @@ pseudocode is not mistaken for non-compliance. | RSA8d | `error.message` contains the callback's text | Wrapped as 40170 with the original in `cause`; `__str__` renders both | Rendering | | CHM2 | Missing metrics default to 0 | `ChannelMetrics.from_dict` uses a bare `obj.get(name)`, so any omitted metric parses as `None` | Open bug, broader than CHM2g/h | | CHM2g, CHM2h | `objectPublishers` and `objectSubscribers` on `ChannelMetrics` | Neither is modelled, so both are dropped on parsing. The test asserts their absence, and turns red once they are added | Open bug | -| TO3l8 | `maxMessageSize` is a client option, default 65536 | Rejected by `Options.__init__`. `ably/realtime/channel.py:422` reads it with `getattr(..., 65536)`, so the default holds but cannot be configured, nor overridden by `connectionDetails` (CD2c) | Open bug | +| TO3l8, CD2c | `maxMessageSize` is a client option, default 65536, which `connectionDetails.maxMessageSize` overrides for realtime publishes | The option and its default match, and REST and realtime publishes both measure per TM6 and raise 40009. `ConnectionDetails.from_dict` does not parse `maxMessageSize`, so a realtime publish is checked against the client option alone. No test asserts CD2c; `test_rtn24_connection_details_override` sends the field and reads back only the details ably-python parses | Open bug, CD2c only | | RTN15, RTN23 | A DISCONNECTED `ErrorInfo` needs no `statusCode` | `ConnectionManager.on_disconnected` evaluates `exception.status_code >= 500` unguarded, so a DISCONNECTED whose error omits `statusCode` raises `TypeError` in a task whose exception is only logged, and the connection silently stays CONNECTED. `DISCONNECTED_MESSAGE` supplies 400 | Open bug | | TO3l4, TO3l5 | `httpRequestTimeout` defaults to 10000 and `httpMaxRetryCount` to 3 on the options object | Both are defaulted on `Options`. `http_max_retry_count` reads back as 3 and matches; `http_request_timeout` reads back as 10, because ably-python's is seconds where the spec's is milliseconds, on the value a caller passes as much as on the default — gated, with the measurement, under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests | The unit is an open bug, recorded there | | RTC7 (TO3l3, TO3l4) | `client.options.httpOpenTimeout == 4000` and `httpRequestTimeout == 10000` | 4 and 10: both are defaulted on `Options`, in seconds where TO3l3 and TO3l4 count milliseconds. The three realtime timeouts the same test checks are milliseconds and match | Open bug, and the unit reaches the wire — `rest/proxy/RSC15l2/timeout-triggers-fallback-0` measures a request outliving its configured timeout by a factor of a thousand. Gated under *`httpRequestTimeout` is seconds where the specification counts milliseconds* in Failing Tests. Moving the HTTP durations to milliseconds is breaking, and is held back from 4.0.0 pending a decision on units across the options | @@ -3535,8 +3539,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable -set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 +it. As of this writing that is 154 failures, 1575 passes and 27 skips with the variable +set, and 181 skips and 1575 passes without it, the 1575 being 1427 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/rest/unit/channel/publish_test.py b/test/uts/rest/unit/channel/publish_test.py index 7ebef9d4..2cca9168 100644 --- a/test/uts/rest/unit/channel/publish_test.py +++ b/test/uts/rest/unit/channel/publish_test.py @@ -11,7 +11,7 @@ from ably.pubsub.types.message import Message from ably.pubsub.util.exceptions import AblyException from test.uts.helpers.client import rest_client -from test.uts.helpers.deviations import deviation, spec_error +from test.uts.helpers.deviations import spec_error from test.uts.helpers.mock_http import MockHttpClient @@ -173,7 +173,6 @@ def on_request(request): # UTS: rest/unit/RSL1i/message-size-limit-0 -@deviation async def test_rsl1i_message_size_limit(): channel_name = f'test-RSL1i-{random_id()}' captured_requests = [] @@ -203,7 +202,10 @@ def on_request(request): captured_requests.clear() request_count = 0 - large_data = 'x' * size + # UTS SPEC ERROR: the spec publishes `"x" * size` as the data, but TM6a counts the + # name too, so its at-limit case is five bytes over. The data is shortened by the + # name's length so that the message is the size each case states. See deviations.md. + large_data = 'x' * (size - len('event')) if expect_success: await channel.publish(name='event', data=large_data) diff --git a/test/uts/rest/unit/types/options_types_test.py b/test/uts/rest/unit/types/options_types_test.py index a0a46c33..93a9c520 100644 --- a/test/uts/rest/unit/types/options_types_test.py +++ b/test/uts/rest/unit/types/options_types_test.py @@ -47,11 +47,7 @@ async def test_to3_client_options_attributes(): assert options.add_request_ids is False assert options.query_time is False - # DEVIATION: TO3l8 `maxMessageSize` (default 65536) is not a client option here. Realtime - # publishes read it with getattr(options, 'max_message_size', 65536), so the 64KiB default - # holds, but it cannot be configured and passing it is rejected. - with pytest.raises(TypeError): - Options(max_message_size=65536) + assert options.max_message_size == 65536 # The HTTP layer applies the defaults the options hold: 10 seconds, and the primary # host plus 3 fallbacks to try (TO3l5). From d6835beee69f863fe855278a8ac692f9dd8dfb62 Mon Sep 17 00:00:00 2001 From: owenpearson Date: Mon, 26 Oct 2026 18:41:11 +0000 Subject: [PATCH 8/8] pubsub: remove the recover client option TO3i's recover option names a recovery key, from which RTN16 has a client inherit the state of an earlier connection. ably-python does not implement connection recovery: it accepted recover, documented it on the realtime factory, and stored it with a property and a setter, but never sent it, so a client given a recovery key silently started a fresh connection. The option, its property and its docstring are removed. The RTN16 and RTC1c tests stay gated, and the RTN16f1 malformed-key test, which passed only because the key was ignored, is gated with them; test/uts/deviations.md records the change and updates its counts. This is a breaking change: passing recover now raises TypeError. Callers should drop it. Connection resumption (RTN15) is unaffected, and connection recovery may return, with the option, in a minor release, as UPDATING.md notes. Co-Authored-By: Claude Opus 5.5 --- UPDATING.md | 19 ++++++++++++++ ably/pubsub/server/__init__.py | 3 --- ably/pubsub/types/options.py | 11 +------- test/uts/deviations.md | 26 ++++++++++--------- .../unit/client/realtime_client_test.py | 4 +-- .../connection/connection_recovery_test.py | 15 ++++++----- 6 files changed, 44 insertions(+), 34 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 801db6dc..3e9085fd 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -534,6 +534,25 @@ your Ably account allows larger messages: pubsub_http_client = create_http_client(key='xxx', max_message_size=131072) ``` +### The `recover` option is removed + +The realtime client accepted a `recover` option, documented as resuming an earlier +connection from its recovery key, but connection recovery is not implemented: the option was +stored and never sent, so a client given one always started a fresh connection. It is gone, +and passing it raises `TypeError`. Remove it, and rely on the client's own resumption of a +dropped connection, which is implemented. Connection recovery may return in a minor release, +along with the `recover` option. + +Example 3.x code: +```python +realtime = AblyRealtime(key='xxx', recover=recovery_key) +``` + +Example 4.0.0 code: +```python +pubsub_realtime_client = create_realtime_client(key='xxx') +``` + ## Version 2.x to 3.0.0 The 3.0.0 version of ably-python introduces several breaking changes to improve the realtime experience and align the API with the Ably specification. These include: diff --git a/ably/pubsub/server/__init__.py b/ably/pubsub/server/__init__.py index d203ae8d..0582dcf4 100644 --- a/ably/pubsub/server/__init__.py +++ b/ably/pubsub/server/__init__.py @@ -194,9 +194,6 @@ def create_realtime_client(**kwargs) -> PubSubRealtimeClient: auto_connect: bool Connect as soon as the client is created. Set this to False to connect later with `connect()`. Defaults to True. - recover: str - A recovery key from a previous connection, used to resume that connection's - state and message continuity rather than starting fresh. queue_messages: bool Hold messages published while the connection is not yet established and send them once it is, rather than failing them. Defaults to True. diff --git a/ably/pubsub/types/options.py b/ably/pubsub/types/options.py index 36597162..4ff30c99 100644 --- a/ably/pubsub/types/options.py +++ b/ably/pubsub/types/options.py @@ -26,7 +26,7 @@ def decode(self, delta: bytes, base: bytes) -> bytes: class Options(AuthOptions): def __init__(self, client_id=None, tls=True, port=0, - tls_port=0, use_binary_protocol=True, queue_messages=True, recover=False, endpoint=None, + tls_port=0, use_binary_protocol=True, queue_messages=True, endpoint=None, http_open_timeout=None, http_request_timeout=None, realtime_request_timeout=None, http_max_retry_count=None, http_max_retry_duration=None, fallback_hosts=None, fallback_retry_timeout=None, disconnected_retry_timeout=None, @@ -87,7 +87,6 @@ def __init__(self, client_id=None, tls=True, port=0, self.__use_binary_protocol = use_binary_protocol self.__queue_messages = queue_messages self.__echo_messages = echo_messages - self.__recover = recover self.__endpoint = endpoint self.__http_open_timeout = http_open_timeout self.__http_request_timeout = http_request_timeout @@ -165,14 +164,6 @@ def queue_messages(self, value): def echo_messages(self): return self.__echo_messages - @property - def recover(self): - return self.__recover - - @recover.setter - def recover(self, value): - self.__recover = value - @property def endpoint(self): return self.__endpoint diff --git a/test/uts/deviations.md b/test/uts/deviations.md index 8ef1f73e..b42aa6b0 100644 --- a/test/uts/deviations.md +++ b/test/uts/deviations.md @@ -33,8 +33,8 @@ parametrized over a table of fixtures the specification gives inline; and five `objects/unit` tests are parametrized, three over a table and two over the scenarios the specification loops through. That turns 1481 derived tests into 1608 pytest cases. -Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1293 Test IDs -(1302 tests, 1427 cases) pass, 152 (152 tests, 154 cases) are gated behind +Of **1472 Test IDs, derived as 1481 tests and run as 1608 pytest cases**: 1292 Test IDs +(1301 tests, 1426 cases) pass, 153 (153 tests, 155 cases) are gated behind `RUN_DEVIATIONS`, and 27 (27 tests, 27 cases) cannot be run at all. The three groups are disjoint: two Test IDs have a gated part and a passing part — `TI1`, as two functions, and `TO/endpoint-affects-host`, as one parametrized test — and are counted with the gated, @@ -45,13 +45,13 @@ the Test IDs come from `uts/rest/unit` (503 tests, 536 cases), 482 from `uts/rest/integration` (84, 122), 73 from `uts/realtime/integration` (73, 95) and 17 from `uts/objects/integration` (17, 29); 8 of the REST integration ids (8, 8), 30 of the realtime ones (30, 30) and 5 of the objects ones (5, 5) come from the `proxy` package -within each. Of the gated Test IDs 69 are REST and 83 realtime, which is 70 REST cases -and 84 realtime. None of the 339 objects Test IDs is gated or unrunnable. +within each. Of the gated Test IDs 69 are REST and 84 realtime, which is 70 REST cases +and 85 realtime. None of the 339 objects Test IDs is gated or unrunnable. A further 148 pytest cases cover the harness itself and are not derived from a specification: 130 under `helpers/`, for the mocks, and 18 under `objects/helpers/`, for the objects builders and the synced-channel mock. -The 141 gated Test IDs that record SDK non-compliance — 141 tests, 143 cases — reduce to +The 142 gated Test IDs that record SDK non-compliance — 142 tests, 144 cases — reduce to **60 distinct root causes**, 20 on the REST side and 40 on the realtime side. Two further defects are recorded below with no test of their own, because the specification's test cannot discriminate (RTP18a) or has nothing to assert against (the timezone split on @@ -1036,15 +1036,17 @@ issued token are all real, and each test reaches the missing call before it fail assertions either side of it are known to hold against real server responses. The `push_channels` file was additionally run against a throwaway shim — a `PushChannel` posting and deleting `/push/channelSubscriptions` with `X-Ably-DeviceToken` — and passes in -full against it. The two recovery tests are a different shape: there is no missing call for -them to reach, so each runs end to end against the sandbox through `uts-proxy` and fails -on the `recover` parameter the connection never sends. +full against it. The two recovery tests reach the missing option rather than a missing +call: each fails building the client that passes `recover`, which raises `TypeError` now the +option is gone, after RTN16d has done its first phase against the sandbox through +`uts-proxy`. Before 4.0.0 removed the option, both ran end to end and failed only on the +`recover` parameter the connection never sent. | Spec points | Missing | Test IDs | |---|---|---| | RSA17, RSA17b–g, BAR2, TRS2, TRF2 | `Auth#revokeTokens`, `TokenRevocationTargetSpecifier` and the two token revocation result types; the `BatchResult` they would arrive in is the batch API's. Gated against `auth.revoke_tokens(targets, issued_before=, allow_reauth_margin=)` returning `success_count` / `failure_count` / `results`, with `target` / `issued_before` / `applies_at` / `error` per result. RSA17d is the one case that needs no server at all — a token-authenticated client must refuse locally with 40162/401 — so it can be satisfied before any of the wire work | 21 | | RSH7, RSH7a–e, RSH6, RSH8 | `PushChannel`: `channel.push`, `client.device`, `LocalDevice`. The push *admin* surface (RSH1) does exist | 12 | -| RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. `recover` is in the `Options` signature, stored, and given a property and a setter (`options.py:30,111,193,196`), and read nowhere. No `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding. Measured through the proxy: a client built with a valid `recover=` opened a `ws_connect` whose query parameters were `{'accessToken': …, 'v': '5'}` — no `recover` — and was given a fresh `connectionId`. RTN16l is otherwise fully compliant, taking the proxy's `recovery-failed-new-id`, `recovery-failed-new-key` and error 80008 and staying CONNECTED; only the absent parameter fails it | 8 | +| RTN16, RTN16f–k, RTC1c (TO3i) | Connection recovery, entire. No `recover` client option — 3.x accepted and stored one and read it nowhere, and 4.0.0 removed it, so passing it raises `TypeError` — no `Connection#createRecoveryKey`, no `recover` connect parameter, no recovery-key decoding. Measured through the proxy before the option was removed: a client built with a valid `recover=` opened a `ws_connect` whose query parameters were `{'accessToken': …, 'v': '5'}` — no `recover` — and was given a fresh `connectionId`. RTN16l was otherwise fully compliant, taking the proxy's `recovery-failed-new-id`, `recovery-failed-new-key` and error 80008 and staying CONNECTED; only the absent parameter failed it. RTN16f1's malformed key passed while the option was ignored, and is gated with the rest now that it cannot be passed | 9 | | RTL22, RTL22a–d, MFI1, MFI2a–e | `MessageFilter`. `RealtimeChannel.subscribe` (`channel.py:262-273`) accepts only a `str` or a callable, and there is no filter type of any shape to spell. Each test builds its filter through the module's `message_filter()` helper, which is the one place to repoint when the type lands | 5 | | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | Derived channels: `DeriveOptions` and `Channels.getDerived`. `grep -r derive ably/` is empty. Each test imports `DeriveOptions` inside its body so the module still loads | 5 | | RTB1, RTB1a, RTB1b | Retry backoff, jitter and `retryIn`. Retry timers schedule the flat configured timeout (`connectionmanager.py:753`, `channel.py:866-871`); `grep` for jitter/backoff/retry_in returns nothing, and neither `ConnectionStateChange` nor `ChannelStateChange` carries `retryIn` | 4 | @@ -2878,7 +2880,7 @@ Each row is one feature and one issue. None is a bug in existing code. | Feature | Spec points | Tests | Reproduction (`-k` against `test/uts/realtime/unit/`) | |---|---|---|---| -| Connection recovery, entire — `createRecoveryKey`, the `recover` connect parameter, recovery-key decoding. `recover` is stored on `Options` and read nowhere. Measured through the proxy: a client given a valid `recover=` sends no `recover` query parameter and is issued a fresh `connectionId` | RTN16, RTN16f–k, RTC1c | 8 | `connection/connection_recovery_test.py`, `client/realtime_client_test.py -k rtc1c`, and `test/uts/realtime/integration/proxy/connection_resume_test.py -k "rtn16d or rtn16l"` | +| Connection recovery, entire — the `recover` client option, `createRecoveryKey`, the `recover` connect parameter, recovery-key decoding. 3.x stored `recover` and read it nowhere, and 4.0.0 removed it. Measured through the proxy before then: a client given a valid `recover=` sent no `recover` query parameter and was issued a fresh `connectionId` | RTN16, RTN16f–k, RTC1c | 9 | `connection/connection_recovery_test.py`, `client/realtime_client_test.py -k rtc1c`, and `test/uts/realtime/integration/proxy/connection_resume_test.py -k "rtn16d or rtn16l"` | | `MessageFilter` and filtered subscriptions | RTL22, RTL22a–d, MFI1, MFI2a–e | 5 | `channels/channel_subscribe_test.py -k rtl22` | | Derived channels — `DeriveOptions`, `Channels.getDerived` | RTS5, RTS5a, RTS5a1, RTS5a2, DO2a | 5 | `channels/channel_options_test.py -k "rts5 or do2a"` | | Retry backoff, jitter and `retryIn` on both state-change types | RTB1, RTB1a, RTB1b | 4 | `connection/backoff_jitter_test.py` | @@ -3539,8 +3541,8 @@ The header states how many derived tests there are, how many pass, how many are gated and how many cannot run. Those numbers are the check that the file is still true: in pytest cases, the gated count must equal the number of failures under `RUN_DEVIATIONS=1`, and gated plus unrunnable must equal the number of skips without -it. As of this writing that is 154 failures, 1575 passes and 27 skips with the variable -set, and 181 skips and 1575 passes without it, the 1575 being 1427 derived cases and 148 +it. As of this writing that is 155 failures, 1574 passes and 27 skips with the variable +set, and 182 skips and 1574 passes without it, the 1574 being 1426 derived cases and 148 harness ones — 130 under `helpers/` and 18 under `objects/helpers/`. Those figures need the `submodules` checkout CI makes: without it, three `rest/unit/encoding` tests that read the ably-common fixtures skip as well. The tiers are told apart by the junit report's diff --git a/test/uts/realtime/unit/client/realtime_client_test.py b/test/uts/realtime/unit/client/realtime_client_test.py index fd9bfa1d..a5f2a005 100644 --- a/test/uts/realtime/unit/client/realtime_client_test.py +++ b/test/uts/realtime/unit/client/realtime_client_test.py @@ -195,8 +195,8 @@ async def test_rtc1b_auto_connect_option(): # UTS: realtime/unit/RTC1c/recover-option-0 -# DEVIATION: the `recover` option is stored and never read, so no `recover` query -# parameter is ever sent. See deviations.md. +# DEVIATION: there is no `recover` client option, so this raises TypeError on +# construction. See deviations.md. @deviation async def test_rtc1c_recover_option(): recovery_key = encode_recovery_key('previous-connection-key', 5, {'channel1': 'serial1'}) diff --git a/test/uts/realtime/unit/connection/connection_recovery_test.py b/test/uts/realtime/unit/connection/connection_recovery_test.py index 9684513c..c862c14b 100644 --- a/test/uts/realtime/unit/connection/connection_recovery_test.py +++ b/test/uts/realtime/unit/connection/connection_recovery_test.py @@ -3,11 +3,10 @@ Spec points: RTN16d, RTN16f, RTN16f1, RTN16g, RTN16g1, RTN16g2, RTN16g3, RTN16i, RTN16j, RTN16k, RTN16l -RTN16 connection recovery is absent from ably-python: `recover` is a client -option with a property and a setter but is read nowhere in the library, there is -no `recover` connect parameter and no recovery key to create or decode. Every -test here but the malformed-key one is therefore gated; see -deviations.md. +RTN16 connection recovery is absent from ably-python: there is no `recover` +client option, no `recover` connect parameter and no recovery key to create or +decode, so passing `recover` raises TypeError. Every test here is therefore +gated; see deviations.md. """ import asyncio @@ -240,6 +239,9 @@ async def test_rtn16f_recover_initializes_msgserial(): # UTS: realtime/unit/RTN16f1/malformed-recovery-key-0 +# DEVIATION: there is no `recover` client option, so this raises TypeError on +# construction. See deviations.md. +@deviation async def test_rtn16f1_malformed_recovery_key(): attempts = [] @@ -259,8 +261,7 @@ def on_connection_attempt(conn): assert client.connection.connection_manager.connection_id == 'fresh-conn' assert client.connection.connection_details.connection_key == 'fresh-key' - # The malformed key reaches the connection parameters no more than a valid - # one would: `recover` is stored and never read + # RTN16f1: a malformed key is dropped, and the connection proceeds without it assert 'recover' not in attempts[0].url.query_params assert 'resume' not in attempts[0].url.query_params assert len(attempts) == 1