From 7d38520c474f8cfd34f581fe507937a411c6540c Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Mon, 17 Aug 2026 10:36:29 +0200 Subject: [PATCH] docs: Rewrite the custom HTTP client guide around the transport-hook contract --- README.md | 4 +- docs/02_concepts/10_custom_http_clients.mdx | 76 ++++++--- docs/02_concepts/code/10_plugging_in_async.py | 27 ++-- docs/02_concepts/code/10_plugging_in_sync.py | 27 ++-- docs/03_guides/05_custom_http_client.mdx | 61 ++++---- .../code/05_custom_http_client_async.py | 142 ++++++++++++----- .../code/05_custom_http_client_sync.py | 148 +++++++++++++----- pyproject.toml | 7 +- 8 files changed, 328 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index ba5b9084..a2884ae2 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ For a guided walkthrough — authenticating, running an Actor, and reading its r - **Tiered timeouts** — short / medium / long tiers picked per endpoint, overridable per call ([Timeouts](https://docs.apify.com/api/client/python/docs/concepts/timeouts)). - **Pagination and streaming** — iterate datasets, key-value store keys, or live logs without manual paging or buffering ([Pagination](https://docs.apify.com/api/client/python/docs/concepts/pagination), [Streaming](https://docs.apify.com/api/client/python/docs/concepts/streaming-resources)). - **Convenience methods** — `call()`, `wait_for_finish()`, nested resource access, and other shortcuts that hide platform quirks ([Convenience methods](https://docs.apify.com/api/client/python/docs/concepts/convenience-methods)). -- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([Custom HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). +- **Pluggable HTTP layer** — swap the default [Impit](https://github.com/apify/impit)-based HTTP client for `httpx`, `requests`, `aiohttp`, or any custom implementation ([HTTP clients](https://docs.apify.com/api/client/python/docs/concepts/custom-http-clients)). - **Structured errors** — every API error surfaces as an [`ApifyApiError`](https://docs.apify.com/api/client/python/reference/class/ApifyApiError) with HTTP-specific subclasses for precise handling ([Error handling](https://docs.apify.com/api/client/python/docs/concepts/error-handling)). - **Debug logging** — opt-in structured logging on the `apify_client` logger captures request URLs, status codes, retry attempts, and more ([Logging](https://docs.apify.com/api/client/python/docs/concepts/logging)). @@ -192,7 +192,7 @@ The full documentation lives at **[docs.apify.com/api/client/python](https://doc | [Introduction](https://docs.apify.com/api/client/python/docs) | Overview, prerequisites, and a tour of the client. | | [Quick start](https://docs.apify.com/api/client/python/docs/quick-start) | Authenticate, run an Actor, and fetch its results step by step. | | [Concepts](https://docs.apify.com/api/client/python/docs/concepts/asyncio-support) | Asyncio, single vs. collection clients, nested clients, error handling, retries, logging, convenience methods, pagination, streaming, custom HTTP clients, timeouts. | -| [Guides](https://docs.apify.com/api/client/python/docs/guides/passing-input-to-actor) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), use HTTPX as the HTTP client. | +| [Guides](https://docs.apify.com/api/client/python/docs/guides/passing-input-to-actor) | Pass input to an Actor, manage tasks for reusable input, retrieve Actor data, integrate with data libraries (e.g. Pandas), build a custom HTTP client. | | [Upgrading](https://docs.apify.com/api/client/python/docs/upgrading/upgrading-to-v3) | Migrating between major versions. | | [API reference](https://docs.apify.com/api/client/python/reference) | Generated reference for every class, method, and model. | | [Changelog](https://docs.apify.com/api/client/python/docs/changelog) | Release history and breaking changes. | diff --git a/docs/02_concepts/10_custom_http_clients.mdx b/docs/02_concepts/10_custom_http_clients.mdx index 0d2e41c1..173b9a7a 100644 --- a/docs/02_concepts/10_custom_http_clients.mdx +++ b/docs/02_concepts/10_custom_http_clients.mdx @@ -1,7 +1,7 @@ --- id: custom-http-clients -title: Custom HTTP clients -description: Replace the default HTTP client with a custom implementation. +title: HTTP clients +description: Understand the built-in HTTP clients and the custom client interface. --- import Tabs from '@theme/Tabs'; @@ -17,7 +17,8 @@ import ArchitectureImportsExample from '!!raw-loader!./code/10_architecture_impo import PluggingInAsyncExample from '!!raw-loader!./code/10_plugging_in_async.py'; import PluggingInSyncExample from '!!raw-loader!./code/10_plugging_in_sync.py'; -The Apify API client uses a pluggable HTTP client architecture. By default, it ships with an [Impit](https://github.com/apify/impit)-based HTTP client that handles retries, timeouts, passing headers, and more. You can replace it with your own implementation for use cases like custom logging, proxying, request modification, or integrating with a different HTTP library. +The Apify API client uses a pluggable HTTP layer. It ships with an [Impit](https://github.com/apify/impit)-based default +and accepts fully custom synchronous or asynchronous implementations. ## Default HTTP client @@ -25,8 +26,8 @@ When you create an `ApifyClient` or `ApifyClient` or `ApifyClientAsync` constructor: @@ -45,33 +46,53 @@ You can configure the default client through the `HttpClient` and `HttpClientAsync` + add the synchronous or asynchronous request pipeline, retry loop, transport hooks, and lifecycle interface. +- The built-in Impit classes inherit directly from the corresponding sync or async class and adapt the underlying + transport. + +`HttpClient.is_timeout_error(exc)` and `HttpClientAsync.is_timeout_error(exc)` provide the public, transport-neutral way +to determine whether an exception is a timeout. Their shared implementation recognizes Python's `TimeoutError`; +transport adapters override it when their HTTP library defines additional timeout exception types. This lets +higher-level features such as streamed logs classify timeouts without depending on Impit or private implementation +details. + +Responses use one separate abstraction: -- `HttpClient` / `HttpClientAsync` - Abstract base classes that define the interface. Extend one of these to create a custom HTTP client by implementing the `call` method. - `HttpResponse` - A [runtime-checkable protocol](https://docs.python.org/3/library/typing.html#typing.runtime_checkable) that defines the expected response shape. Any object with the required attributes and methods satisfies the protocol — no inheritance needed. To plug in your custom implementation, use the `ApifyClient.with_custom_http_client` class method. -All of these are available as top-level imports from the `apify_client` package: +The built-in Impit classes are thin transport adapters over the request implementation in `HttpClient` and +`HttpClientAsync`. Custom transport adapters implement the request, error-classification, and lifecycle hooks. They +inherit request construction, retries, timeout growth, API error conversion, logging, and statistics from the base. + +All of these are available from the `apify_client.http_clients` module: {ArchitectureImportsExample} -### The call method +### The transport contract -The `call` method receives all the information needed to make an HTTP request: +The public `call` method provides the shared request pipeline. A concrete transport implements these hooks: -- `method` - HTTP method (`GET`, `POST`, `PUT`, `DELETE`, etc.). -- `url` - Full URL to make the request to. -- `headers` - Additional headers to include. -- `params` - Query parameters to append to the URL. -- `data` - Raw request body (mutually exclusive with `json`). -- `json` - JSON-serializable request body (mutually exclusive with `data`). -- `stream` - Whether to stream the response body. -- `timeout` - Timeout for the request as a `timedelta`. +- `send_request(...)` sends one prepared request and returns an `HttpResponse`. The inherited `call` needs it, so + every transport adapter has to implement it. +- `is_retryable_transport_error(exc)` classifies transport failures for the shared retry loop. The default classifies + nothing as retryable, so a transport that skips it gives up on the first connection failure. +- `is_timeout_error(exc)` identifies transport-specific timeout exceptions for higher-level client features. The + default recognizes Python's `TimeoutError`. Timeout classification is independent of retryability, so a timeout + the retry loop should retry has to be listed in `is_retryable_transport_error` too. +- `close()` or `aclose()` closes resources owned by the transport. The default does nothing, which is correct for a + transport that owns no pool or session. -It must return an object satisfying the `HttpResponse` protocol. +The `@override` decorators in the built-in Impit adapters make these implementations explicit and allow type checkers +to catch misspelled or incompatible overrides. ### The HTTP response protocol @@ -93,6 +114,10 @@ It must return an object satisfying the `HttpRe :::note Many HTTP libraries, including our default [Impit](https://github.com/apify/impit) or for example [HTTPX](https://www.python-httpx.org/) already satisfy this protocol out of the box. + +For a streamed response, consume the body inside its context manager with `iter_bytes()` / `aiter_bytes()`, or call +`read()` / `aread()` before accessing `content`. Some transports, including HTTPX, intentionally reject `content` on +an unread streamed response. ::: ### Plugging it in @@ -115,18 +140,23 @@ Use the `ApifyClient.wit After that, all API calls made through the client will go through your custom HTTP client. :::warning -When using a custom HTTP client, you are responsible for constructing the request, handling retries, timeouts, and errors yourself. The default retry logic is not applied. +If you override `call` itself, your implementation becomes responsible for request preparation, retries, timeouts, API +error conversion, logging, and statistics. Implementing the transport hooks and inheriting `call` keeps the shared +behavior. ::: ## Use cases -Custom HTTP clients might be useful when you need to: +Custom HTTP clients might be useful when the built-in Impit clients don't cover your requirements, for example when +you need to: -- **Use a different HTTP library** - Swap Impit for [httpx](https://www.python-httpx.org/), [requests](https://requests.readthedocs.io/), or [aiohttp](https://docs.aiohttp.org/). +- **Use a different HTTP library** - Integrate [httpx](https://www.python-httpx.org/), [requests](https://requests.readthedocs.io/), [aiohttp](https://docs.aiohttp.org/), or another transport. - **Route through a proxy** - Add proxy support or request routing. - **Implement custom retry logic** - Use different backoff strategies or retry conditions. - **Log requests and responses** - Track API calls for debugging or auditing. - **Modify requests** - Add custom fields, modify the body, or change headers. - **Collect custom metrics** - Measure request latency, track error rates, or count API calls. -For a step-by-step walkthrough of building a custom HTTP client, see the [Using HTTPX as the HTTP client](/api/client/python/docs/guides/custom-http-client-httpx) guide. +For complete synchronous and asynchronous implementations over a transport with a different response API, see +[Build a custom HTTP client](../03_guides/05_custom_http_client.mdx). You can also refer to the +`HttpClient` API reference for the synchronous contract. diff --git a/docs/02_concepts/code/10_plugging_in_async.py b/docs/02_concepts/code/10_plugging_in_async.py index 331f0b62..dad83733 100644 --- a/docs/02_concepts/code/10_plugging_in_async.py +++ b/docs/02_concepts/code/10_plugging_in_async.py @@ -1,8 +1,7 @@ -from typing import Any +from typing_extensions import override from apify_client import ApifyClientAsync from apify_client.http_clients import HttpClientAsync, HttpResponse -from apify_client.types import Timeout TOKEN = 'MY-APIFY-TOKEN' @@ -10,18 +9,26 @@ class MyHttpClientAsync(HttpClientAsync): """Custom async HTTP client.""" - async def call( + @override + async def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', - ) -> HttpResponse: ... + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> HttpResponse: + """Send one request through the custom transport.""" + raise NotImplementedError + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # List the transport's transient failures here, e.g. its timeout + # and connection errors. Returning False for everything opts out + # of transport retries entirely. + return isinstance(exc, TimeoutError) async def main() -> None: diff --git a/docs/02_concepts/code/10_plugging_in_sync.py b/docs/02_concepts/code/10_plugging_in_sync.py index 386281ae..8f0aa362 100644 --- a/docs/02_concepts/code/10_plugging_in_sync.py +++ b/docs/02_concepts/code/10_plugging_in_sync.py @@ -1,8 +1,7 @@ -from typing import Any +from typing_extensions import override from apify_client import ApifyClient from apify_client.http_clients import HttpClient, HttpResponse -from apify_client.types import Timeout TOKEN = 'MY-APIFY-TOKEN' @@ -10,18 +9,26 @@ class MyHttpClient(HttpClient): """Custom sync HTTP client.""" - def call( + @override + def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', - ) -> HttpResponse: ... + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, + ) -> HttpResponse: + """Send one request through the custom transport.""" + raise NotImplementedError + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + # List the transport's transient failures here, e.g. its timeout + # and connection errors. Returning False for everything opts out + # of transport retries entirely. + return isinstance(exc, TimeoutError) def main() -> None: diff --git a/docs/03_guides/05_custom_http_client.mdx b/docs/03_guides/05_custom_http_client.mdx index b7314a1c..97594279 100644 --- a/docs/03_guides/05_custom_http_client.mdx +++ b/docs/03_guides/05_custom_http_client.mdx @@ -1,39 +1,46 @@ --- -id: custom-http-client-httpx -title: Use HTTPX as the HTTP client -description: Replace the default Impit HTTP client with one based on HTTPX. +id: custom-http-client +title: Build a custom HTTP client +description: Implement the HTTP client contract with AIOHTTP and requests. --- import ApiLink from '@theme/ApiLink'; +import CodeBlock from '@theme/CodeBlock'; import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -import CodeBlock from '@theme/CodeBlock'; import CustomHttpClientAsyncExample from '!!raw-loader!./code/05_custom_http_client_async.py'; import CustomHttpClientSyncExample from '!!raw-loader!./code/05_custom_http_client_sync.py'; -This guide shows how to replace the default `ImpitHttpClient` and `ImpitHttpClientAsync` with one based on [HTTPX](https://www.python-httpx.org/). The same approach works for any HTTP library — see [Custom HTTP clients](/api/client/python/docs/concepts/custom-http-clients) for the underlying architecture. - -## Why HTTPX? +This guide implements a custom `HttpClientAsync` with +[AIOHTTP](https://docs.aiohttp.org/) and a custom `HttpClient` with +[requests](https://requests.readthedocs.io/). Neither library satisfies the +`HttpResponse` protocol, so both examples also show how to adapt a +foreign response API. -You might want to use [HTTPX](https://www.python-httpx.org/) instead of the default [Impit](https://github.com/apify/impit)-based client for reasons like: +For an overview of the architecture and the built-in Impit implementation, see +[HTTP clients](../02_concepts/10_custom_http_clients.mdx). -- You already use HTTPX in your project and want a single HTTP stack. -- You need HTTPX-specific features. -- You want fine-grained control over connection pooling or proxy routing. - -## Implementation +## Installation -The implementation involves two steps: +Install the transport alongside the Apify client. Neither AIOHTTP nor requests is an `apify-client` extra: -1. **Extend `HttpClient` (sync) or `HttpClientAsync` (async)** and implement the `call` method that delegates to HTTPX. -2. **Pass it to `ApifyClient.with_custom_http_client`** to create a client that uses your implementation. +```bash +pip install apify-client aiohttp # for the asynchronous client +pip install apify-client requests # for the synchronous client +``` -The `call` method receives parameters like `method`, `url`, `headers`, `params`, `data`, `json`, `stream`, and `timeout`. Map them to the corresponding HTTPX arguments — most map directly, except `data` which becomes HTTPX's `content` parameter and `timeout` which needs conversion from `timedelta` to seconds. +## Implementation -A convenient property of HTTPX is that its `httpx.Response` object already satisfies the `HttpResponse` protocol, so you can return it directly without wrapping. +Each example has three parts: -One part of the contract isn't visible in the method signature: `call` must raise `ApifyApiError` for error responses instead of returning them. The resource clients rely on that error to work correctly. For example, `get` methods translate a 404 raised this way into a `None` return value, and user code handling `ApifyApiError` keeps working. +1. The response adapter, `AiohttpResponse` or `RequestsResponse`, maps the library's own response onto the + `HttpResponse` protocol that resource clients expect. +2. The client, `AiohttpHttpClient` or `RequestsHttpClient`, implements the transport, error-classification, + timeout-classification, and lifecycle hooks. It inherits request preparation, retry handling, timeout growth, + and API error conversion from its base class. +3. `with_custom_http_client()` connects the implementation to the resource clients and applies the API token. The + context manager closes the session at shutdown. @@ -49,16 +56,8 @@ One part of the contract isn't visible in the method signature: `call` must rais :::warning -When using a custom HTTP client, you are responsible for handling retries, timeouts, and error handling yourself. The built-in retry logic with exponential backoff is part of the default `ImpitHttpClient` and is not applied to custom implementations. +These are compact integration examples, not a replacement for all built-in client behavior. A production custom +client should account for transport-specific details such as proxy configuration, TLS settings, redirects, and +response resource cleanup. The shared base provides retries, logging, statistics, timeout growth, and API error +conversion. ::: - -## Going further - -The example above is minimal on purpose. In a production setup, you might want to extend it with: - -- **Retry logic** - Use [HTTPX's event hooks](https://www.python-httpx.org/advanced/event-hooks/) or utilize library like [tenacity](https://tenacity.readthedocs.io/) to retry failed requests. -- **Custom headers** - You can add headers in the `call` method before delegating to HTTPX. -- **Connection lifecycle** - Close the underlying `httpx.Client` when done by adding a `close()` method to your custom client. -- **Proxy support** - You can pass `proxy=...` when creating the `httpx.Client`. -- **Metrics collection** - Track request latency, error rates, or other metrics by adding instrumentation in the `call` method. -- **Logging** - Log requests and responses for debugging or auditing purposes. diff --git a/docs/03_guides/code/05_custom_http_client_async.py b/docs/03_guides/code/05_custom_http_client_async.py index 44c81080..efd1481c 100644 --- a/docs/03_guides/code/05_custom_http_client_async.py +++ b/docs/03_guides/code/05_custom_http_client_async.py @@ -1,75 +1,137 @@ from __future__ import annotations import asyncio -from http import HTTPStatus +import json as jsonlib from typing import TYPE_CHECKING, Any -import httpx +import aiohttp +from typing_extensions import override from apify_client import ApifyClientAsync -from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpClientAsync, HttpResponse if TYPE_CHECKING: - from apify_client.types import Timeout + from collections.abc import AsyncIterator, Iterator, Mapping TOKEN = 'MY-APIFY-TOKEN' -class HttpxClientAsync(HttpClientAsync): - """Custom async HTTP client using HTTPX library.""" +class AiohttpResponse: + """Adapt an aiohttp response to the Apify client's HttpResponse protocol.""" + + def __init__(self, response: aiohttp.ClientResponse) -> None: + self._response = response + self._body: bytes | None = None + + @property + def status_code(self) -> int: + return self._response.status + + @property + def headers(self) -> Mapping[str, str]: + return self._response.headers + + @property + def content(self) -> bytes: + if self._body is None: + raise RuntimeError( + 'The streamed response has not been read yet; ' + 'use aread() or aiter_bytes()' + ) + return self._body + + @property + def text(self) -> str: + encoding = self._response.charset or 'utf-8' + return self.content.decode(encoding, errors='replace') + + def json(self) -> Any: + return jsonlib.loads(self.text) + + def read(self) -> bytes: + return self.content + + async def aread(self) -> bytes: + if self._body is None: + self._body = await self._response.read() + return self._body + + def close(self) -> None: + self._response.close() + + async def aclose(self) -> None: + self._response.release() + await self._response.wait_for_close() + + def iter_bytes(self) -> Iterator[bytes]: + body = self.content + if body: + yield body + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + if self._body is not None: + if self._body: + yield self._body + return + async for chunk in self._response.content.iter_chunked(64 * 1024): + yield chunk + + +class AiohttpHttpClient(HttpClientAsync): + """Minimal custom asynchronous HTTP client backed by aiohttp.""" def __init__(self) -> None: super().__init__() - self._client = httpx.AsyncClient() + self._session = aiohttp.ClientSession() + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance( + exc, aiohttp.ServerTimeoutError + ) + + @override + async def aclose(self) -> None: + await self._session.close() - async def call( + @override + async def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, ) -> HttpResponse: - timeout_secs = self._compute_timeout(timeout, attempt=1) or 0 - - # Merge the client's default headers (including authorization) - # with the per-request ones. - headers = self._merge_headers(self._headers, headers) - - response = await self._client.request( + response = await self._session.request( method=method, url=url, headers=headers, - params=params, - content=data, - json=json, - timeout=timeout_secs, + data=content, + timeout=aiohttp.ClientTimeout(total=timeout), ) + adapted_response = AiohttpResponse(response) - # Raising `ApifyApiError` for error responses is part of the `call` - # contract. The resource clients rely on it, e.g. to translate a 404 - # into a `None` return value of `get` methods. - if response.status_code >= HTTPStatus.BAD_REQUEST: - raise ApifyApiError(response, attempt=1, method=method) + if not stream: + await adapted_response.aread() - # httpx.Response satisfies the HttpResponse protocol, - # so it can be returned directly. - return response + return adapted_response + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance(exc, (TimeoutError, aiohttp.ClientError)) -async def main() -> None: - client = ApifyClientAsync.with_custom_http_client( - token=TOKEN, - http_client=HttpxClientAsync(), - ) - actor = await client.actor('apify/hello-world').get() - print(actor) +async def main() -> None: + async with AiohttpHttpClient() as http_client: + client = ApifyClientAsync.with_custom_http_client( + token=TOKEN, + http_client=http_client, + ) + actor = await client.actor('apify/hello-world').get() + print(actor) if __name__ == '__main__': diff --git a/docs/03_guides/code/05_custom_http_client_sync.py b/docs/03_guides/code/05_custom_http_client_sync.py index c6716256..996dcef5 100644 --- a/docs/03_guides/code/05_custom_http_client_sync.py +++ b/docs/03_guides/code/05_custom_http_client_sync.py @@ -1,74 +1,138 @@ from __future__ import annotations -from http import HTTPStatus +import json as jsonlib from typing import TYPE_CHECKING, Any -import httpx +import requests +from typing_extensions import override from apify_client import ApifyClient -from apify_client.errors import ApifyApiError from apify_client.http_clients import HttpClient, HttpResponse if TYPE_CHECKING: - from apify_client.types import Timeout + from collections.abc import AsyncIterator, Iterator, Mapping TOKEN = 'MY-APIFY-TOKEN' -class HttpxClient(HttpClient): - """Custom HTTP client using HTTPX library.""" +class RequestsResponse: + """Adapt a requests response to the Apify client's HttpResponse protocol.""" + + def __init__(self, response: requests.Response) -> None: + self._response = response + self._body: bytes | None = None + + @property + def status_code(self) -> int: + return self._response.status_code + + @property + def headers(self) -> Mapping[str, str]: + return self._response.headers + + @property + def content(self) -> bytes: + if self._body is None: + raise RuntimeError( + 'The streamed response has not been read yet; use read() or iter_bytes()' + ) + return self._body + + @property + def text(self) -> str: + encoding = self._response.encoding or 'utf-8' + return self.content.decode(encoding, errors='replace') + + def json(self) -> Any: + return jsonlib.loads(self.text) + + def read(self) -> bytes: + if self._body is None: + self._body = self._response.content + return self._body + + async def aread(self) -> bytes: + return self.read() + + def close(self) -> None: + self._response.close() + + async def aclose(self) -> None: + self.close() + + def iter_bytes(self) -> Iterator[bytes]: + if self._body is not None: + if self._body: + yield self._body + return + yield from self._response.iter_content(64 * 1024) + + async def aiter_bytes(self) -> AsyncIterator[bytes]: + for chunk in self.iter_bytes(): + yield chunk + + +class RequestsHttpClient(HttpClient): + """Minimal custom synchronous HTTP client backed by requests.""" def __init__(self) -> None: super().__init__() - self._client = httpx.Client() + self._session = requests.Session() + + @override + def is_timeout_error(self, exc: Exception) -> bool: + return super().is_timeout_error(exc) or isinstance(exc, requests.Timeout) - def call( + @override + def close(self) -> None: + self._session.close() + + @override + def send_request( self, *, method: str, url: str, - headers: dict[str, str] | None = None, - params: dict[str, Any] | None = None, - data: str | bytes | bytearray | None = None, - json: Any = None, - stream: bool | None = None, - timeout: Timeout = 'medium', + headers: dict[str, str], + content: bytes | None, + timeout: float | None, + stream: bool, ) -> HttpResponse: - timeout_secs = self._compute_timeout(timeout, attempt=1) or 0 - - # Merge the client's default headers (including authorization) - # with the per-request ones. - headers = self._merge_headers(self._headers, headers) - - response = self._client.request( + response = self._session.request( method=method, url=url, headers=headers, - params=params, - content=data, - json=json, - timeout=timeout_secs, + data=content, + timeout=timeout, + stream=stream, + ) + adapted_response = RequestsResponse(response) + + if not stream: + adapted_response.read() + + return adapted_response + + @override + def is_retryable_transport_error(self, exc: Exception) -> bool: + return isinstance( + exc, + ( + requests.ConnectionError, + requests.Timeout, + requests.exceptions.ChunkedEncodingError, + ), ) - - # Raising `ApifyApiError` for error responses is part of the `call` - # contract. The resource clients rely on it, e.g. to translate a 404 - # into a `None` return value of `get` methods. - if response.status_code >= HTTPStatus.BAD_REQUEST: - raise ApifyApiError(response, attempt=1, method=method) - - # httpx.Response satisfies the HttpResponse protocol, - # so it can be returned directly. - return response def main() -> None: - client = ApifyClient.with_custom_http_client( - token=TOKEN, - http_client=HttpxClient(), - ) - - actor = client.actor('apify/hello-world').get() - print(actor) + with RequestsHttpClient() as http_client: + client = ApifyClient.with_custom_http_client( + token=TOKEN, + http_client=http_client, + ) + actor = client.actor('apify/hello-world').get() + print(actor) if __name__ == '__main__': diff --git a/pyproject.toml b/pyproject.toml index 0019741f..55ff2619 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -208,12 +208,7 @@ include = ["docs/**/*.py", "website/**/*.py"] unresolved-import = "ignore" [[tool.ty.overrides]] -include = ["docs/**/10_plugging_in_async.py", "docs/**/10_plugging_in_sync.py"] -[tool.ty.overrides.rules] -empty-body = "ignore" - -[[tool.ty.overrides]] -include = ["docs/**/05_custom_http_client_async.py", "docs/**/05_custom_http_client_sync.py"] +include = ["docs/**/05_custom_http_client_async.py"] [tool.ty.overrides.rules] invalid-argument-type = "ignore"