From 44cd45014ce24163a615b0798737beee40dca7fd Mon Sep 17 00:00:00 2001 From: Sean O'Brien <60306702+stobrien89@users.noreply.github.com> Date: Wed, 9 Sep 2026 11:18:12 -0400 Subject: [PATCH] enhancement(http): add schema-cached HTTP binding serde --- designs/http-binding-serde.md | 496 ++++++++++++++++++ .../smithy-core/src/smithy_core/schemas.py | 50 +- .../smithy-core/tests/unit/test_schemas.py | 35 +- .../src/smithy_http/aio/protocols.py | 30 +- .../src/smithy_http/deserializers.py | 46 +- .../src/smithy_http/schema_extensions.py | 96 ++++ .../src/smithy_http/serializers.py | 352 ++++++++----- .../tests/unit/aio/test_protocols.py | 144 ++++- .../smithy-http/tests/unit/test_bindings.py | 53 +- .../tests/unit/test_serializers.py | 139 ++++- 10 files changed, 1250 insertions(+), 191 deletions(-) create mode 100644 designs/http-binding-serde.md create mode 100644 packages/smithy-http/src/smithy_http/schema_extensions.py diff --git a/designs/http-binding-serde.md b/designs/http-binding-serde.md new file mode 100644 index 000000000..277cba685 --- /dev/null +++ b/designs/http-binding-serde.md @@ -0,0 +1,496 @@ +# HTTP Binding Serialization and Deserialization + +## Status + +Draft. + +## Summary + +`HTTPBindingSerializer` routes generated input members to HTTP headers, URI +components, and payloads. `HTTPResponseDeserializer` reads response bindings. +Both use schema-cached `HTTPBindingSchemaMetadata`. Each schema derives and +caches its binding map once. + +Generated structures keep the existing `serialize()`, `serialize_members()`, +and `deserialize()` interfaces. HTTP protocols call `serialize_members()` +directly for generated inputs and use `serialize()` as a compatibility path +for handwritten inputs. Document codecs continue to handle document payloads. +The change requires no generator updates. + +### Request flow + +```text +Generated input.serialize_members() + | + v + HTTPBindingSerializer + | + cached member route table + / | \ + v v v + HTTP fields URI payload codec + \ | / + v + HTTPRequest +``` + +### Response flow + +```text +HTTPResponse + | + v +HTTPResponseDeserializer + | +cached headers/status/payload/body bindings + | | + v v +location deserializers payload codec + | | + +------------+-------------+ + v + generated deserialize() +``` + +## Motivation + +The current request and response paths construct binding matchers for each +serde operation. Response deserialization also walks every structure member +before it delegates document members to the payload codec. Generated schemas +remain stable during client execution, so the runtime can store these derived +bindings on each schema. + +Calling `serialize_members()` directly removes the root `begin_struct()` call +for generated inputs. Cached binding metadata removes repeated trait +classification and matcher allocation. Existing serializer, deserializer, and +codec contracts remain unchanged. + +## Scope + +* Lazy extension caching on `Schema`. +* Cached request and response HTTP binding metadata. +* Generic request serialization through `HTTPBindingSerializer`. +* Cached response binding lookup in `HTTPResponseDeserializer` and + `HTTPResponseSerializer`. +* A `serialize()` compatibility path for handwritten inputs. + +JSON, XML, and query codecs keep their current interfaces. Generated +deserialization keeps its callback contract. Structure construction hooks and +filtered document schemas require separate designs. + +## Existing Generated Interface + +Generated structures provide the required runtime interface: + +```python +class ExampleInput: + SCHEMA: ClassVar[Schema] + + def serialize(self, serializer: ShapeSerializer) -> None: + with serializer.begin_struct(self.SCHEMA) as struct_serializer: + self.serialize_members(struct_serializer) + + def serialize_members(self, serializer: ShapeSerializer) -> None: + ... +``` + +`serialize()` remains the general entry point for serializing a complete shape. +`serialize_members()` remains the efficient entry point for a runtime that has +already opened or otherwise established the containing structure. + +HTTP request serialization uses `serialize_members()` because the HTTP binding +serializer owns the request and document-body structure state. Other callers +can continue to use `serialize()`. + +Generated deserialization remains: + +```python +class ExampleOutput: + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: + kwargs: dict[str, Any] = {} + deserializer.read_struct(cls.SCHEMA, consumer=...) + return cls(**kwargs) +``` + +`HTTPResponseDeserializer` implements this existing callback-based interface. + +## Schema Extensions + +### Core API + +A schema extension is a shared, typed descriptor with a provider: + +```python +@dataclass(frozen=True, slots=True, eq=False) +class SchemaExtension[T]: + provider: Callable[[Schema], T] +``` + +Schemas expose: + +```python +class Schema: + def get_extension[T](self, extension: SchemaExtension[T]) -> T: + ... +``` + +The HTTP runtime creates one extension descriptor: + +```python +HTTP_BINDING_SCHEMA_EXTENSION = SchemaExtension( + _build_http_binding_schema_metadata +) +``` + +Every HTTP protocol instance uses the module-level descriptor. Each schema +caches one `HTTPBindingSchemaMetadata` value for that descriptor. + +### Cache Requirements + +`Schema` allocates its extension dictionary on first use and keys entries by +descriptor identity. Providers publish complete values. Concurrent cache +misses may invoke a provider more than once, which avoids a lock on the lookup +path. + +The cache is an implementation attribute. It stays out of schema equality, +representation, `dataclasses.fields()`, `dataclasses.asdict()`, and the +generated constructor. `dataclasses.replace()` creates a schema with an empty +cache. + +Extension values use immutable containers. `HTTPBindingSchemaMetadata` is a +frozen, slotted dataclass whose collections are tuples. + +### HTTP Binding Metadata + +The HTTP extension stores both request and response metadata: + +```python +@dataclass(frozen=True, slots=True) +class HTTPBindingSchemaMetadata: + request_bindings: tuple[Binding, ...] + response_bindings: tuple[Binding, ...] + + has_request_body: bool + has_response_body: bool + + payload_member: Schema | None + event_stream_member: Schema | None + + response_bound_members: tuple[ + tuple[Schema, Binding, str | None, bool], ... + ] + response_status: int +``` + +`request_bindings` and `response_bindings` are indexed by +`Schema.member_index`. They are separate because a trait such as `@httpQuery` +is a request binding but is treated as a document-body member if the same +structure is used as an output. + +Response-bound entries retain schema-member order and precompute: + +* The member schema. +* The response binding. +* The normalized lowercase header name. +* Whether the member is list-valued. + +The name field stores the prefix for prefix-header entries and is `None` for +status and payload bindings. This ordered tuple avoids scanning body members +while preserving the existing deserializer consumer order. + +Callers use `HTTPBindingSchemaMetadata` instead of repeating trait inspection. +New fields require a benchmark that identifies repeated work in a serde path. + +### Document Body Metadata + +The extension records whether request and response structures contain document +body members. Document codecs continue to receive the original structure +schema. + +The original schema preserves codec behavior and keeps one effective schema per +Smithy shape ID. Filtered document schemas require an explicit schema identity +contract for codecs. + +## Request Serialization + +### `HTTPBindingSerializer` + +`HTTPBindingSerializer` is a structure-member serializer and request builder: + +```python +class HTTPBindingSerializer(InterceptingSerializer): + def __init__( + self, + *, + payload_codec: Codec, + schema: Schema, + http_trait: HTTPTrait, + endpoint_trait: EndpointTrait | None = None, + omit_empty_payload: bool = True, + ) -> None: + ... + + def build_request(self) -> HTTPRequest: + ... +``` + +The constructor: + +1. Gets the cached `HTTPBindingSchemaMetadata`. +2. Creates serializers for headers, query parameters, path labels, and host + labels. +3. Selects the payload mode: + * Event stream. + * Raw `@httpPayload`. + * Structured `@httpPayload`. + * Implicit document body. +4. Opens the document-body structure when required. + +Generated `serialize_members()` calls the normal `ShapeSerializer` methods. +`HTTPBindingSerializer.before()` performs an indexed route lookup and returns +the serializer for that binding: + +```python +def before(self, schema: Schema) -> ShapeSerializer: + binding = self._binding_metadata.request_bindings[ + schema.expect_member_index() + ] + match binding: + case Binding.HEADER | Binding.PREFIX_HEADERS: + return self.header_serializer + case Binding.QUERY | Binding.QUERY_PARAMS: + return self.query_serializer + case Binding.LABEL: + return self.path_serializer + case Binding.HOST: + return self.host_prefix_serializer + case _: + return self._payload_serializer +``` + +`build_request()`: + +* Closes an implicit document-body structure. +* Resolves the final payload stream. +* Adds the payload content type and known content length. +* Resolves the host prefix, path, and query string. +* Returns an `HTTPRequest`. + +If member serialization raises, `abort()` closes the document serializer with +the active exception information. + +### Protocol Integration + +The generated fast path is: + +```python +serializer = HTTPBindingSerializer( + payload_codec=self.payload_codec, + schema=operation.input_schema, + http_trait=operation.schema.expect_trait(HTTPTrait), + endpoint_trait=operation.schema.get_trait(EndpointTrait), +) +input.serialize_members(serializer) +return serializer.build_request() +``` + +The protocol still supports handwritten `SerializeableShape` +implementations that do not implement `SerializeableStruct`: + +```python +if isinstance(input, SerializeableStruct): + input.serialize_members(serializer) + return serializer.build_request() + +legacy = HTTPRequestSerializer(...) +input.serialize(legacy) +if legacy.result is None: + raise ExpectationNotMetError("Expected a serialized HTTP request.") +return legacy.result +``` + +Generated operation inputs are structures and use the direct path. Handwritten +inputs that only implement `serialize()` use `HTTPRequestSerializer`. + +### Compatibility Facade + +`HTTPRequestSerializer` remains available with its existing constructor and +`result` behavior. Its `begin_struct()` implementation delegates to +`HTTPBindingSerializer`. + +Callers can continue to instantiate `HTTPRequestSerializer` or pass it to a +generated shape's `serialize()` method. + +## Response Deserialization + +### `HTTPResponseDeserializer` + +`HTTPResponseDeserializer` keeps its public name and constructor. It implements +`ShapeDeserializer` and uses cached binding metadata. + +`read_struct()` performs three steps: + +1. Gets the cached `HTTPBindingSchemaMetadata`. +2. Reads precomputed non-body response bindings in schema-member order. +3. Delegates the original response schema to the payload codec when document + body members are present. + +The cached tuple contains only transport-bound members. `read_struct()` visits +those members directly, then delegates document members to the payload codec. +Location-specific `ShapeDeserializer` implementations continue to read scalar +and collection values. + +## Response Serialization + +`HTTPResponseSerializer` uses the same `HTTPBindingSchemaMetadata`: + +* `response_bindings` route members. +* `has_response_body` determines whether the payload codec is invoked. +* `response_status` provides the modeled default. +* Request and response serialization share payload and event-stream metadata. + +`HTTPResponseSerializer` retains its public name and constructor. + +`HTTPResponseBindingSerializer` is an implementation helper and consumes the +cached binding metadata directly. + +## Payload Modes + +### Implicit Document Body + +The HTTP binding serializer routes document-body members to the serializer +created by the payload codec. The codec receives the original structure schema. + +`omit_empty_payload` controls whether the serializer writes an empty body. + +### Explicit Payload + +String, enum, and blob payloads use raw payload serialization. Their default +content types are: + +| Shape | Content type | +|---|---| +| String or enum | `text/plain` | +| Blob | `application/octet-stream` | + +`@mediaType` overrides the default. + +The payload codec serializes and deserializes aggregate payloads with the +payload member schema. + +### Streaming Payload + +A streaming blob passes through without buffering. For `@requiresLength`, the +runtime uses an existing `Content-Length` field or calls `tell()` and `seek()` +on a synchronous stream. It raises `SerializationError` when neither source +provides a length. + +### Event Stream + +Event-stream payloads retain the existing writable/readable async body +behavior. Event message serialization and deserialization remain the +responsibility of the event-stream runtime. + +## Compatibility + +Generated models already provide `serialize()`, `serialize_members()`, +`deserialize()`, and schemas with stable member indexes. The protocol selects a +different existing method, so generated output stays unchanged. + +| Caller | Expected result | +|---|---| +| Generated SDK input | Uses `serialize_members()` fast path | +| Generated SDK output | Uses existing `deserialize()` callback path | +| Handwritten shape with both serialization methods | Uses fast path | +| Handwritten shape with only `serialize()` | Uses compatibility facade | +| Direct `HTTPRequestSerializer` user | Existing API remains available | +| Direct `HTTPRequestBindingSerializer` user | Existing constructor remains available | +| Direct `HTTPResponseDeserializer` user | Existing name uses the cached implementation | + +Document codecs receive the original root schema. This preserves wire behavior +for responses whose document body also contains transport-bound members. + +## Relationship to smithy-java and smithy-php + +smithy-java and smithy-php separate HTTP location routing from document codec +logic and derive binding knowledge from model metadata. Smithy Python uses +`SchemaExtension` for that metadata and keeps its existing +`ShapeSerializer` and `ShapeDeserializer` contracts. + +Python member schemas carry `Schema.member_index`, which indexes request and +response route tables. Generated deserialization keeps the consumer callback +instead of filling a positional member buffer. + +## Performance + +The Rest JSON benchmark ran on an x86 benchmark instance. It uses the +`AwsSdkPerformanceBenchmarkModels` artifacts and exercises the complete client +protocol path. Each case used 5,000 warmup iterations and 10,000 measured +iterations. + +| Group | Geometric mean p50 change | +|---|---:| +| Request serialization | 49.7% faster | +| Response deserialization | 45.2% faster | +| All 10 Rest JSON cases | 47.5% faster | + +| Case | Before | After | Change | +|---|---:|---:|---:| +| Serialize CopyObject baseline | 33.94 us | 13.85 us | 59.2% faster | +| Serialize CopyObject M | 109.66 us | 87.23 us | 20.5% faster | +| Serialize PutObject S | 38.72 us | 17.89 us | 53.8% faster | +| Serialize PutObject M | 39.35 us | 18.18 us | 53.8% faster | +| Serialize PutObject L | 38.91 us | 18.06 us | 53.6% faster | +| Deserialize CopyObject baseline | 24.52 us | 12.26 us | 50.0% faster | +| Deserialize CopyObject M | 68.39 us | 51.71 us | 24.4% faster | +| Deserialize GetObject S | 56.43 us | 29.00 us | 48.6% faster | +| Deserialize GetObject M | 57.17 us | 28.33 us | 50.4% faster | +| Deserialize GetObject L | 56.82 us | 29.05 us | 48.9% faster | + +The direct `serialize_members()` call removes the root structure wrapper. +Cached route tables remove matcher construction. Ordered response metadata +visits transport-bound members directly and stores normalized header names. + +## Validation + +* Package tests: 1,994 passed and 9 skipped. +* Generated AWS JSON 1.0, AWS JSON 1.1, AWS Query, and Rest JSON protocol suites + passed. +* Ruff and Pyright passed. +* All Python packages built. +* All 64 serde artifact cases passed validation. + +## Alternatives + +### Rebuild Matchers for Each Operation + +This keeps the current control flow but repeats trait classification and schema +walks. The x86 Rest JSON benchmark measures the cost removed by cached metadata. + +### Generate Protocol Serde + +Generated HTTP routing duplicates protocol logic in each client and increases +generated package size. Shared runtime components keep routing behavior in one +package. + +### Cache Metadata on Codec Instances + +A codec-local cache duplicates schema metadata across codec instances and +requires the runtime to manage each cache's lifetime. The schema-local cache +shares one value across protocol instances. + +### Store HTTP Fields Directly on `Schema` + +HTTP fields on `Schema` couple smithy-core to HTTP transport concerns. Typed +extensions keep the core schema transport-neutral. + +## Future Work + +* Add schema-attached target type and structure construction hooks. +* Deserialize into positional member buffers before constructing structures. +* Add timestamp or collection metadata when a benchmark identifies repeated + formatting work. +* Define codec schema identity before adding filtered document schemas. +* Apply schema extensions to JSON, XML, and query codecs. +* Audit external matcher usage before changing matcher visibility. diff --git a/packages/smithy-core/src/smithy_core/schemas.py b/packages/smithy-core/src/smithy_core/schemas.py index 3719bde32..c7ec4b20a 100644 --- a/packages/smithy-core/src/smithy_core/schemas.py +++ b/packages/smithy-core/src/smithy_core/schemas.py @@ -1,8 +1,17 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from collections.abc import Mapping, Sequence +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field, replace -from typing import TYPE_CHECKING, Any, NotRequired, Required, Self, TypedDict, overload +from typing import ( + TYPE_CHECKING, + Any, + NotRequired, + Required, + Self, + TypedDict, + cast, + overload, +) from .exceptions import ExpectationNotMetError, SmithyError from .shapes import ShapeID, ShapeType @@ -95,6 +104,8 @@ def __init__( if member_index is not None: object.__setattr__(self, "member_index", member_index) + object.__setattr__(self, "_extensions", None) + @property def member_name(self) -> str | None: """The name of the member, if the shape is the MEMBER type.""" @@ -174,6 +185,33 @@ def expect_trait(self, t: "type[Trait] | ShapeID") -> "Trait | DynamicTrait": id = t if isinstance(t, ShapeID) else t.id return self.traits[id] + def get_extension[T](self, extension: "SchemaExtension[T]") -> T: + """Get or lazily build metadata associated with this schema. + + Extension descriptors are intended to be shared across all codec and protocol + instances. Values are cached per schema after construction. Concurrent cache + misses may construct the same value more than once, but subsequent calls return + the published cached value. + + :param extension: The shared extension descriptor. + :returns: The cached extension value for this schema. + """ + extensions = cast( + "dict[object, Any] | None", + getattr(self, "_extensions", None), + ) + if extensions is None: + value = extension.provider(self) + object.__setattr__(self, "_extensions", {extension: value}) + return value + + try: + return extensions[extension] + except KeyError: + value = extension.provider(self) + extensions[extension] = value + return value + def __contains__(self, item: Any): """Returns whether the schema has the given member or trait.""" match item: @@ -271,6 +309,14 @@ def member( ) +@dataclass(frozen=True, slots=True, eq=False) +class SchemaExtension[T]: + """A shared provider of lazily cached schema metadata.""" + + provider: Callable[[Schema], T] + """Build the extension value for a schema.""" + + class MemberSchema(TypedDict): """A simplified schema for members. diff --git a/packages/smithy-core/tests/unit/test_schemas.py b/packages/smithy-core/tests/unit/test_schemas.py index c6b835228..28cc625a3 100644 --- a/packages/smithy-core/tests/unit/test_schemas.py +++ b/packages/smithy-core/tests/unit/test_schemas.py @@ -1,11 +1,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from dataclasses import replace +from dataclasses import asdict, fields, replace from typing import Any import pytest from smithy_core.exceptions import ExpectationNotMetError -from smithy_core.schemas import Schema +from smithy_core.schemas import Schema, SchemaExtension from smithy_core.shapes import ShapeID, ShapeType from smithy_core.traits import ( DynamicTrait, @@ -47,6 +47,37 @@ def test_get_unknown_trait_by_id(): assert schema.get_trait(SensitiveTrait.id) is None +def test_schema_extension_is_built_once_and_cached() -> None: + calls = 0 + + def build_extension(schema: Schema) -> tuple[ShapeID, int]: + nonlocal calls + calls += 1 + return schema.id, calls + + extension = SchemaExtension(build_extension) + schema = Schema(id=ID, shape_type=ShapeType.STRUCTURE) + + first = schema.get_extension(extension) + second = schema.get_extension(extension) + + assert first == (ID, 1) + assert second is first + assert calls == 1 + + +def test_schema_extension_cache_is_not_dataclass_state() -> None: + extension = SchemaExtension(lambda schema: schema) + schema = Schema(id=ID, shape_type=ShapeType.STRUCTURE) + + assert schema.get_extension(extension) is schema + assert "_extensions" not in {schema_field.name for schema_field in fields(schema)} + assert "_extensions" not in asdict(schema) + + replaced = replace(schema) + assert replaced.get_extension(extension) is replaced + + def test_members_list(): member_name = "baz" member = Schema( diff --git a/packages/smithy-http/src/smithy_http/aio/protocols.py b/packages/smithy-http/src/smithy_http/aio/protocols.py index 6bd37c17d..cde19d9f8 100644 --- a/packages/smithy-http/src/smithy_http/aio/protocols.py +++ b/packages/smithy-http/src/smithy_http/aio/protocols.py @@ -22,12 +22,12 @@ from smithy_core.interfaces import StreamingBlob as SyncStreamingBlob from smithy_core.prelude import DOCUMENT from smithy_core.schemas import APIOperation -from smithy_core.serializers import SerializeableShape +from smithy_core.serializers import SerializeableShape, SerializeableStruct from smithy_core.shapes import ShapeID from smithy_core.traits import EndpointTrait, HTTPTrait from ..deserializers import HTTPResponseDeserializer -from ..serializers import HTTPRequestSerializer +from ..serializers import HTTPBindingSerializer, HTTPRequestSerializer from .interfaces import HTTPErrorIdentifier, HTTPRequest, HTTPResponse @@ -98,22 +98,31 @@ def serialize_request[ endpoint: URI, context: TypedProperties, ) -> HTTPRequest: - # TODO(optimization): request binding cache like done in SJ + if isinstance(input, SerializeableStruct): + serializer = HTTPBindingSerializer( + payload_codec=self.payload_codec, + schema=operation.input_schema, + http_trait=operation.schema.expect_trait(HTTPTrait), + endpoint_trait=operation.schema.get_trait(EndpointTrait), + ) + try: + input.serialize_members(serializer) + except BaseException as error: + serializer.abort(type(error), error, error.__traceback__) + raise + return serializer.build_request() + serializer = HTTPRequestSerializer( payload_codec=self.payload_codec, http_trait=operation.schema.expect_trait(HTTPTrait), endpoint_trait=operation.schema.get_trait(EndpointTrait), ) - - input.serialize(serializer=serializer) - request = serializer.result - - if request is None: + input.serialize(serializer) + if serializer.result is None: raise ExpectationNotMetError( "Expected request to be serialized, but was None" ) - - return request + return serializer.result async def deserialize_response[ OperationInput: "SerializeableShape", @@ -142,7 +151,6 @@ async def deserialize_response[ if not operation.output_stream_member and not is_streaming_blob(body): body = await self._buffer_async_body(response.body) - # TODO(optimization): response binding cache like done in SJ deserializer = HTTPResponseDeserializer( payload_codec=self.payload_codec, http_trait=operation.schema.expect_trait(HTTPTrait), diff --git a/packages/smithy-http/src/smithy_http/deserializers.py b/packages/smithy-http/src/smithy_http/deserializers.py index 61674098c..7813bd498 100644 --- a/packages/smithy-http/src/smithy_http/deserializers.py +++ b/packages/smithy-http/src/smithy_http/deserializers.py @@ -16,8 +16,6 @@ from smithy_core.schemas import Schema from smithy_core.shapes import ShapeType from smithy_core.traits import ( - HTTPHeaderTrait, - HTTPPrefixHeadersTrait, HTTPTrait, MediaTypeTrait, TimestampFormatTrait, @@ -26,8 +24,9 @@ from smithy_core.utils import ensure_utc, strict_parse_bool, strict_parse_float from .aio.interfaces import HTTPResponse -from .bindings import Binding, ResponseBindingMatcher +from .bindings import Binding from .interfaces import Field, Fields +from .schema_extensions import HTTP_BINDING_SCHEMA_EXTENSION from .utils import split_header if TYPE_CHECKING: @@ -39,7 +38,7 @@ class HTTPResponseDeserializer(SpecificShapeDeserializer): - """Binds :py:class:`HTTPResponse` properties to a DeserializableShape.""" + """Deserialize HTTP response bindings through the shape deserializer contract.""" # Note: caller will have to read the body if it's async and not streaming def __init__( @@ -66,40 +65,43 @@ def __init__( def read_struct( self, schema: Schema, consumer: Callable[[Schema, ShapeDeserializer], None] ) -> None: - binding_matcher = ResponseBindingMatcher(schema) + binding_metadata = schema.get_extension(HTTP_BINDING_SCHEMA_EXTENSION) - for member in schema.members.values(): - match binding_matcher.match(member): + for member, binding, name, is_list in binding_metadata.response_bound_members: + match binding: case Binding.HEADER: - trait = member.expect_trait(HTTPHeaderTrait) - header = self._response.fields.entries.get(trait.key.lower()) + assert name is not None # noqa: S101 + header = self._response.fields.entries.get(name) if header is not None: - if member.shape_type is ShapeType.LIST: + if is_list: consumer(member, HTTPHeaderListDeserializer(header)) else: - consumer(member, HTTPHeaderDeserializer(header.as_string())) + consumer( + member, + HTTPHeaderDeserializer(header.as_string()), + ) case Binding.PREFIX_HEADERS: - trait = member.expect_trait(HTTPPrefixHeadersTrait) + assert name is not None # noqa: S101 consumer( member, - HTTPHeaderMapDeserializer(self._response.fields, trait.prefix), + HTTPHeaderMapDeserializer(self._response.fields, name), ) case Binding.STATUS: consumer( - member, HTTPResponseCodeDeserializer(self._response.status) + member, + HTTPResponseCodeDeserializer(self._response.status), ) case Binding.PAYLOAD: - if binding_matcher.event_stream_member is None: - assert binding_matcher.payload_member is not None # noqa: S101 - if self._should_read_payload(binding_matcher.payload_member): - deserializer = self._create_payload_deserializer( - binding_matcher.payload_member - ) - consumer(binding_matcher.payload_member, deserializer) + if ( + binding_metadata.event_stream_member is None + and self._should_read_payload(member) + ): + deserializer = self._create_payload_deserializer(member) + consumer(member, deserializer) case _: pass - if binding_matcher.has_body and not self._has_empty_body( + if binding_metadata.has_response_body and not self._has_empty_body( self._response, self._body ): deserializer = self._create_body_deserializer() diff --git a/packages/smithy-http/src/smithy_http/schema_extensions.py b/packages/smithy-http/src/smithy_http/schema_extensions.py new file mode 100644 index 000000000..5e63766fd --- /dev/null +++ b/packages/smithy-http/src/smithy_http/schema_extensions.py @@ -0,0 +1,96 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +from dataclasses import dataclass + +from smithy_core.schemas import Schema, SchemaExtension +from smithy_core.shapes import ShapeType +from smithy_core.traits import HTTPHeaderTrait, HTTPPrefixHeadersTrait + +from .bindings import Binding, RequestBindingMatcher, ResponseBindingMatcher + +type ResponseBoundMember = tuple[Schema, Binding, str | None, bool] + + +@dataclass(frozen=True, slots=True) +class HTTPBindingSchemaMetadata: + """Precomputed request and response HTTP binding metadata for a schema.""" + + request_bindings: tuple[Binding, ...] + """Request binding route for each member index.""" + + response_bindings: tuple[Binding, ...] + """Response binding route for each member index.""" + + has_request_body: bool + """Whether the request structure contains document-body members.""" + + has_response_body: bool + """Whether the response structure contains document-body members.""" + + payload_member: Schema | None + """Member bound to the complete HTTP payload, if present.""" + + event_stream_member: Schema | None + """Member bound to an event stream, if present.""" + + response_bound_members: tuple[ResponseBoundMember, ...] + """Non-body response bindings in schema-member order.""" + + response_status: int + """Default response status derived from the structure traits.""" + + def should_write_request_body(self, omit_empty_payload: bool) -> bool: + """Return whether a request document body should be opened.""" + return self.has_request_body or ( + not omit_empty_payload and self.payload_member is None + ) + + def should_write_response_body(self, omit_empty_payload: bool) -> bool: + """Return whether a response document body should be opened.""" + return self.has_response_body or ( + not omit_empty_payload and self.payload_member is None + ) + + +def _build_http_binding_schema_metadata(schema: Schema) -> HTTPBindingSchemaMetadata: + request_matcher = RequestBindingMatcher(schema) + response_matcher = ResponseBindingMatcher(schema) + members = tuple(schema.members.values()) + request_bindings = tuple(request_matcher.bindings) + response_bindings = tuple(response_matcher.bindings) + + response_bound_members: list[ResponseBoundMember] = [] + for member, binding in zip(members, response_bindings, strict=True): + match binding: + case Binding.HEADER: + trait = member.expect_trait(HTTPHeaderTrait) + response_bound_members.append( + ( + member, + binding, + trait.key.lower(), + member.shape_type is ShapeType.LIST, + ) + ) + case Binding.PREFIX_HEADERS: + trait = member.expect_trait(HTTPPrefixHeadersTrait) + response_bound_members.append((member, binding, trait.prefix, False)) + case Binding.STATUS | Binding.PAYLOAD: + response_bound_members.append((member, binding, None, False)) + case _: + pass + + return HTTPBindingSchemaMetadata( + request_bindings=request_bindings, + response_bindings=response_bindings, + has_request_body=request_matcher.has_body, + has_response_body=response_matcher.has_body, + payload_member=request_matcher.payload_member, + event_stream_member=request_matcher.event_stream_member, + response_bound_members=tuple(response_bound_members), + response_status=response_matcher.response_status, + ) + + +HTTP_BINDING_SCHEMA_EXTENSION = SchemaExtension(_build_http_binding_schema_metadata) +"""Shared HTTP binding extension used by every HTTP protocol instance.""" diff --git a/packages/smithy-http/src/smithy_http/serializers.py b/packages/smithy-http/src/smithy_http/serializers.py index 0f7dc83cf..62ccc1f21 100644 --- a/packages/smithy-http/src/smithy_http/serializers.py +++ b/packages/smithy-http/src/smithy_http/serializers.py @@ -2,11 +2,12 @@ # SPDX-License-Identifier: Apache-2.0 from base64 import b64encode from collections.abc import Callable, Iterator, Sized -from contextlib import contextmanager +from contextlib import AbstractContextManager, contextmanager from datetime import datetime from decimal import Decimal from inspect import iscoroutinefunction from io import BytesIO +from types import TracebackType from typing import TYPE_CHECKING from urllib.parse import quote as urlquote @@ -39,14 +40,22 @@ from .aio import HTTPRequest as _HTTPRequest from .aio import HTTPResponse as _HTTPResponse from .aio.interfaces import HTTPRequest, HTTPResponse -from .bindings import Binding, RequestBindingMatcher, ResponseBindingMatcher +from .bindings import Binding, RequestBindingMatcher +from .schema_extensions import ( + HTTP_BINDING_SCHEMA_EXTENSION, + HTTPBindingSchemaMetadata, +) from .utils import join_query_params if TYPE_CHECKING: from smithy_core.aio.interfaces import StreamingBlob as AsyncStreamingBlob -__all__ = ["HTTPRequestSerializer", "HTTPResponseSerializer"] +__all__ = [ + "HTTPBindingSerializer", + "HTTPRequestSerializer", + "HTTPResponseSerializer", +] # TODO: refactor this to share code with response serializer @@ -81,153 +90,225 @@ def __init__( @contextmanager def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: - payload: AsyncBytesReader | AsyncBytesProvider - binding_serializer: HTTPRequestBindingSerializer + binding_serializer = HTTPBindingSerializer( + payload_codec=self._payload_codec, + schema=schema, + http_trait=self._http_trait, + endpoint_trait=self._endpoint_trait, + omit_empty_payload=self._omit_empty_payload, + ) + try: + yield binding_serializer + except BaseException as error: + binding_serializer.abort(type(error), error, error.__traceback__) + raise + else: + self.result = binding_serializer.build_request() - host_prefix = "" - if self._endpoint_trait is not None: - host_prefix = self._endpoint_trait.host_prefix - content_type = self._payload_codec.media_type - content_length: int | None = None - content_length_required = False +def _compute_content_length( + payload: object, +) -> int | None: + if (tell := getattr(payload, "tell", None)) is not None and not iscoroutinefunction( + tell + ): + start: int = tell() + if (end := _seek(payload, 0, 2)) is not None: + content_length: int = end - start + _seek(payload, start, 0) + return content_length + return None - binding_matcher = RequestBindingMatcher(schema) - if binding_matcher.event_stream_member is not None: - payload = AsyncBytesProvider() - content_type = "application/vnd.amazon.eventstream" - binding_serializer = HTTPRequestBindingSerializer( - SpecificShapeSerializer(), - self._http_trait.path, - host_prefix, - binding_matcher, - ) - yield binding_serializer - elif (payload_member := binding_matcher.payload_member) is not None: - content_length_required = RequiresLengthTrait in payload_member + +def _seek(payload: object, pos: int, whence: int = 0) -> int | None: + if (seek := getattr(payload, "seek", None)) is not None and not iscoroutinefunction( + seek + ): + return seek(pos, whence) + return None + + +class HTTPBindingSerializer(InterceptingSerializer): + """Serialize structure members into HTTP request binding locations. + + Generated structures drive this serializer directly through + :py:meth:`SerializeableStruct.serialize_members`. Call + :py:meth:`build_request` after all members have been written. + """ + + def __init__( + self, + *, + payload_codec: Codec, + schema: Schema, + http_trait: HTTPTrait, + endpoint_trait: EndpointTrait | None = None, + omit_empty_payload: bool = True, + ) -> None: + """Initialize an HTTPBindingSerializer. + + :param payload_codec: The codec used to serialize document-bound members. + :param schema: The structure schema whose members are being serialized. + :param http_trait: The HTTP trait of the operation being handled. + :param endpoint_trait: The optional endpoint trait of the operation. + :param omit_empty_payload: Whether an empty document payload should be omitted. + """ + self._http_trait = http_trait + self._binding_metadata: HTTPBindingSchemaMetadata = schema.get_extension( + HTTP_BINDING_SCHEMA_EXTENSION + ) + self._body_context: AbstractContextManager[ShapeSerializer] | None = None + self._sync_payload: BytesIO | None = None + self._raw_payload_serializer: RawPayloadSerializer | None = None + self._payload: AsyncBytesReader | AsyncBytesProvider | None = None + self._content_type: str | None = payload_codec.media_type + self._content_length: int | None = None + self._content_length_required = False + self._writes_document_body = False + self._result: HTTPRequest | None = None + + host_prefix = endpoint_trait.host_prefix if endpoint_trait is not None else "" + + if self._binding_metadata.event_stream_member is not None: + self._payload = AsyncBytesProvider() + self._content_type = "application/vnd.amazon.eventstream" + payload_serializer: ShapeSerializer = SpecificShapeSerializer() + elif (payload_member := self._binding_metadata.payload_member) is not None: + self._content_length_required = RequiresLengthTrait in payload_member if payload_member.shape_type in ( ShapeType.BLOB, ShapeType.STRING, ShapeType.ENUM, ): if (media_type := payload_member.get_trait(MediaTypeTrait)) is not None: - content_type = media_type.value + self._content_type = media_type.value elif payload_member.shape_type is ShapeType.BLOB: - content_type = "application/octet-stream" + self._content_type = "application/octet-stream" else: - content_type = "text/plain" + self._content_type = "text/plain" - payload_serializer = RawPayloadSerializer() - binding_serializer = HTTPRequestBindingSerializer( - payload_serializer, - self._http_trait.path, - host_prefix, - binding_matcher, - ) - yield binding_serializer - if isinstance(payload_serializer.payload, Sized): - content_length = len(payload_serializer.payload) - payload = AsyncBytesReader(payload_serializer.payload or b"") + self._raw_payload_serializer = RawPayloadSerializer() + payload_serializer = self._raw_payload_serializer else: if (media_type := payload_member.get_trait(MediaTypeTrait)) is not None: - content_type = media_type.value - sync_payload = BytesIO() - payload_serializer = self._payload_codec.create_serializer(sync_payload) - binding_serializer = HTTPRequestBindingSerializer( - payload_serializer, - self._http_trait.path, - host_prefix, - binding_matcher, - ) - yield binding_serializer - content_length = sync_payload.tell() - sync_payload.seek(0) - payload = AsyncBytesReader(sync_payload) + self._content_type = media_type.value + self._sync_payload = BytesIO() + payload_serializer = payload_codec.create_serializer(self._sync_payload) else: - sync_payload = BytesIO() - payload_serializer = self._payload_codec.create_serializer(sync_payload) - if binding_matcher.should_write_body(self._omit_empty_payload): - with payload_serializer.begin_struct(schema) as body_serializer: - binding_serializer = HTTPRequestBindingSerializer( - body_serializer, - self._http_trait.path, - host_prefix, - binding_matcher, - ) - yield binding_serializer - content_length = sync_payload.tell() + self._sync_payload = BytesIO() + payload_serializer = payload_codec.create_serializer(self._sync_payload) + self._writes_document_body = ( + self._binding_metadata.should_write_request_body(omit_empty_payload) + ) + if self._writes_document_body: + self._body_context = payload_serializer.begin_struct(schema) + payload_serializer = self._body_context.__enter__() else: - content_type = None - content_length = None - binding_serializer = HTTPRequestBindingSerializer( - payload_serializer, - self._http_trait.path, - host_prefix, - binding_matcher, - ) - yield binding_serializer - sync_payload.seek(0) - payload = AsyncBytesReader(sync_payload) + self._content_type = None - headers = binding_serializer.header_serializer.headers - if content_type is not None and not any( + self._payload_serializer = payload_serializer + self.header_serializer = HTTPHeaderSerializer() + self.query_serializer = HTTPQuerySerializer() + self.path_serializer = HTTPPathSerializer(http_trait.path) + self.host_prefix_serializer = HostPrefixSerializer( + payload_serializer, host_prefix + ) + + def before(self, schema: Schema) -> ShapeSerializer: + binding = self._binding_metadata.request_bindings[schema.expect_member_index()] + match binding: + case Binding.HEADER | Binding.PREFIX_HEADERS: + return self.header_serializer + case Binding.QUERY | Binding.QUERY_PARAMS: + return self.query_serializer + case Binding.LABEL: + return self.path_serializer + case Binding.HOST: + return self.host_prefix_serializer + case _: + return self._payload_serializer + + def after(self, schema: Schema) -> None: + pass + + def build_request(self) -> HTTPRequest: + """Build the HTTP request after payload serialization.""" + if self._result is not None: + return self._result + + if self._body_context is not None: + self._body_context.__exit__(None, None, None) + self._body_context = None + + payload = self._payload + if self._raw_payload_serializer is not None: + raw_payload = self._raw_payload_serializer.payload + if isinstance(raw_payload, Sized): + self._content_length = len(raw_payload) + payload = AsyncBytesReader(raw_payload or b"") + elif self._sync_payload is not None: + if ( + self._binding_metadata.payload_member is not None + or self._writes_document_body + ): + self._content_length = self._sync_payload.tell() + self._sync_payload.seek(0) + payload = AsyncBytesReader(self._sync_payload) + + assert payload is not None # noqa: S101 + + headers = self.header_serializer.headers + if self._content_type is not None and not any( name.lower() == "content-type" for name, _ in headers ): - headers.append(("content-type", content_type)) + headers.append(("content-type", self._content_type)) - if content_length is not None: - headers.append(("content-length", str(content_length))) + if self._content_length is not None: + headers.append(("content-length", str(self._content_length))) fields = tuples_to_fields(headers) - if content_length_required and "content-length" not in fields: + if self._content_length_required and "content-length" not in fields: content_length = _compute_content_length(payload) if content_length is None: raise SerializationError( - "This operation requires the the content length of the input " + "This operation requires the content length of the input " "stream, but it was not provided and was unable to be computed." ) fields.set_field(Field(name="content-length", values=[str(content_length)])) - self.result = _HTTPRequest( + self._result = _HTTPRequest( method=self._http_trait.method, destination=URI( - host=binding_serializer.host_prefix_serializer.host_prefix, - path=binding_serializer.path_serializer.path, + host=self.host_prefix_serializer.host_prefix, + path=self.path_serializer.path, query=join_query_params( - params=binding_serializer.query_serializer.query_params, + params=self.query_serializer.query_params, prefix=self._http_trait.query or "", ), ), fields=fields, body=payload, ) + return self._result - -def _compute_content_length( - payload: AsyncBytesReader | AsyncBytesProvider, -) -> int | None: - if (tell := getattr(payload, "tell", None)) is not None and not iscoroutinefunction( - tell - ): - start: int = tell() - if (end := _seek(payload, 0, 2)) is not None: - content_length: int = end - start - _seek(payload, start, 0) - return content_length - return None - - -def _seek( - payload: AsyncBytesReader | AsyncBytesProvider, pos: int, whence: int = 0 -) -> None: - if (seek := getattr(payload, "seek", None)) is not None and not iscoroutinefunction( - seek - ): - seek(pos, whence) + def abort( + self, + exc_type: type[BaseException], + exc_value: BaseException, + traceback: TracebackType | None, + ) -> None: + """Abort an in-progress document payload after serialization fails.""" + if self._body_context is not None: + self._body_context.__exit__(exc_type, exc_value, traceback) + self._body_context = None class HTTPRequestBindingSerializer(InterceptingSerializer): - """Delegates HTTP request bindings to binding-location-specific serializers.""" + """Legacy request binding router. + + New code should use :py:class:`HTTPBindingSerializer`, which also owns payload + setup and request finalization. + """ def __init__( self, @@ -236,13 +317,6 @@ def __init__( host_prefix_pattern: str, binding_matcher: RequestBindingMatcher, ) -> None: - """Initialize an HTTPRequestBindingSerializer. - - :param payload_serializer: The :py:class:`ShapeSerializer` to use to serialize - the payload, if necessary. - :param path_pattern: The pattern used to construct the path. - :host_prefix_pattern: The pattern used to construct the host prefix. - """ self._payload_serializer = payload_serializer self.header_serializer = HTTPHeaderSerializer() self.query_serializer = HTTPQuerySerializer() @@ -297,17 +371,18 @@ def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: content_type: str | None = self._payload_codec.media_type content_length: int | None = None + content_length_source: object | None = None content_length_required = False - binding_matcher = ResponseBindingMatcher(schema) - if binding_matcher.event_stream_member is not None: + binding_metadata = schema.get_extension(HTTP_BINDING_SCHEMA_EXTENSION) + if binding_metadata.event_stream_member is not None: payload = AsyncBytesProvider() content_type = "application/vnd.amazon.eventstream" binding_serializer = HTTPResponseBindingSerializer( - SpecificShapeSerializer(), binding_matcher + SpecificShapeSerializer(), binding_metadata ) yield binding_serializer - elif (payload_member := binding_matcher.payload_member) is not None: + elif (payload_member := binding_metadata.payload_member) is not None: content_length_required = RequiresLengthTrait in payload_member if payload_member.shape_type in (ShapeType.BLOB, ShapeType.STRING): if (media_type := payload_member.get_trait(MediaTypeTrait)) is not None: @@ -318,19 +393,21 @@ def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: content_type = "text/plain" payload_serializer = RawPayloadSerializer() binding_serializer = HTTPResponseBindingSerializer( - payload_serializer, binding_matcher + payload_serializer, binding_metadata ) yield binding_serializer - if isinstance(payload_serializer.payload, Sized): - content_length = len(payload_serializer.payload) - payload = AsyncBytesReader(payload_serializer.payload or b"") + raw_payload = payload_serializer.payload + if isinstance(raw_payload, Sized): + content_length = len(raw_payload) + content_length_source = raw_payload + payload = AsyncBytesReader(raw_payload or b"") else: if (media_type := payload_member.get_trait(MediaTypeTrait)) is not None: content_type = media_type.value sync_payload = BytesIO() payload_serializer = self._payload_codec.create_serializer(sync_payload) binding_serializer = HTTPResponseBindingSerializer( - payload_serializer, binding_matcher + payload_serializer, binding_metadata ) yield binding_serializer content_length = sync_payload.tell() @@ -339,12 +416,10 @@ def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: else: sync_payload = BytesIO() payload_serializer = self._payload_codec.create_serializer(sync_payload) - if binding_matcher.should_write_body(self._omit_empty_payload): - if binding_matcher.event_stream_member is not None: - content_type = "application/vnd.amazon.eventstream" + if binding_metadata.should_write_response_body(self._omit_empty_payload): with payload_serializer.begin_struct(schema) as body_serializer: binding_serializer = HTTPResponseBindingSerializer( - body_serializer, binding_matcher + body_serializer, binding_metadata ) yield binding_serializer content_length = sync_payload.tell() @@ -353,7 +428,7 @@ def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: content_length = None binding_serializer = HTTPResponseBindingSerializer( payload_serializer, - binding_matcher, + binding_metadata, ) yield binding_serializer sync_payload.seek(0) @@ -370,23 +445,25 @@ def begin_struct(self, schema: Schema) -> Iterator[ShapeSerializer]: fields = tuples_to_fields(headers) if content_length_required and "content-length" not in fields: - content_length = _compute_content_length(payload) + content_length = _compute_content_length( + content_length_source if content_length_source is not None else payload + ) if content_length is None: raise SerializationError( - "This operation requires the the content length of the input " + "This operation requires the content length of the input " "stream, but it was not provided and was unable to be computed." ) fields.set_field(Field(name="content-length", values=[str(content_length)])) status = binding_serializer.response_code_serializer.response_code if status is None: - if binding_matcher.response_status > 0: - status = binding_matcher.response_status + if binding_metadata.response_status > 0: + status = binding_metadata.response_status else: status = self._http_trait.code self.result = _HTTPResponse( - fields=tuples_to_fields(binding_serializer.header_serializer.headers), + fields=fields, body=payload, status=status, ) @@ -398,7 +475,7 @@ class HTTPResponseBindingSerializer(InterceptingSerializer): def __init__( self, payload_serializer: ShapeSerializer, - binding_matcher: ResponseBindingMatcher, + binding_metadata: HTTPBindingSchemaMetadata, ) -> None: """Initialize an HTTPResponseBindingSerializer. @@ -408,10 +485,11 @@ def __init__( self._payload_serializer = payload_serializer self.header_serializer = HTTPHeaderSerializer() self.response_code_serializer = HTTPResponseCodeSerializer() - self._binding_matcher = binding_matcher + self._binding_metadata = binding_metadata def before(self, schema: Schema) -> ShapeSerializer: - match self._binding_matcher.match(schema): + binding = self._binding_metadata.response_bindings[schema.expect_member_index()] + match binding: case Binding.HEADER | Binding.PREFIX_HEADERS: return self.header_serializer case Binding.STATUS: diff --git a/packages/smithy-http/tests/unit/aio/test_protocols.py b/packages/smithy-http/tests/unit/aio/test_protocols.py index cda2a79f8..8efc7713a 100644 --- a/packages/smithy-http/tests/unit/aio/test_protocols.py +++ b/packages/smithy-http/tests/unit/aio/test_protocols.py @@ -1,21 +1,34 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -from typing import Any +from typing import Any, Self import pytest from smithy_core import URI +from smithy_core.codecs import Codec +from smithy_core.deserializers import ShapeDeserializer from smithy_core.documents import TypeRegistry from smithy_core.endpoints import Endpoint -from smithy_core.interfaces import TypedProperties +from smithy_core.interfaces import TypedProperties as TypedPropertiesInterface from smithy_core.interfaces import URI as URIInterface -from smithy_core.schemas import APIOperation -from smithy_core.shapes import ShapeID +from smithy_core.schemas import APIOperation, Schema +from smithy_core.serializers import ShapeSerializer +from smithy_core.shapes import ShapeID, ShapeType +from smithy_core.traits import HTTPTrait +from smithy_core.types import TypedProperties from smithy_http import Fields from smithy_http.aio import HTTPRequest -from smithy_http.aio.interfaces import HTTPRequest as HTTPRequestInterface -from smithy_http.aio.interfaces import HTTPResponse as HTTPResponseInterface -from smithy_http.aio.protocols import HttpClientProtocol +from smithy_http.aio.interfaces import ( + HTTPErrorIdentifier, +) +from smithy_http.aio.interfaces import ( + HTTPRequest as HTTPRequestInterface, +) +from smithy_http.aio.interfaces import ( + HTTPResponse as HTTPResponseInterface, +) +from smithy_http.aio.protocols import HttpBindingClientProtocol, HttpClientProtocol +from smithy_json import JSONCodec class MockProtocol(HttpClientProtocol): @@ -31,7 +44,7 @@ def serialize_request( operation: APIOperation[Any, Any], input: Any, endpoint: URIInterface, - context: TypedProperties, + context: TypedPropertiesInterface, ) -> HTTPRequestInterface: raise Exception("This is only for tests.") @@ -42,11 +55,124 @@ def deserialize_response( request: HTTPRequestInterface, response: HTTPResponseInterface, error_registry: TypeRegistry, - context: TypedProperties, + context: TypedPropertiesInterface, ) -> Any: raise Exception("This is only for tests.") +class MockBindingProtocol(HttpBindingClientProtocol): + _id = ShapeID("ns.foo#binding") + _codec = JSONCodec() + _error_identifier = HTTPErrorIdentifier() + + @property + def id(self) -> ShapeID: + return self._id + + @property + def payload_codec(self) -> Codec: + return self._codec + + @property + def content_type(self) -> str: + return "application/json" + + @property + def error_identifier(self) -> HTTPErrorIdentifier: + return self._error_identifier + + +class LegacyInput: + SCHEMA = Schema.collection(id=ShapeID("ns.foo#LegacyInput")) + + def __init__(self) -> None: + self.serialize_called = False + + def serialize(self, serializer: ShapeSerializer) -> None: + self.serialize_called = True + with serializer.begin_struct(self.SCHEMA): + pass + + +class StructInput: + SCHEMA = Schema.collection(id=ShapeID("ns.foo#StructInput")) + + def __init__(self) -> None: + self.serialize_members_called = False + + def serialize(self, serializer: ShapeSerializer) -> None: + raise AssertionError("The structure fast path must not call serialize().") + + def serialize_members(self, serializer: ShapeSerializer) -> None: + self.serialize_members_called = True + + +class MockOutput: + SCHEMA = Schema.collection(id=ShapeID("ns.foo#MockOutput")) + + @classmethod + def deserialize(cls, deserializer: ShapeDeserializer) -> Self: + return cls() + + +def test_http_binding_protocol_falls_back_to_legacy_serialize() -> None: + operation = APIOperation( + input=LegacyInput, + output=MockOutput, + schema=Schema( + id=ShapeID("ns.foo#LegacyOperation"), + shape_type=ShapeType.OPERATION, + traits=[HTTPTrait({"method": "POST", "code": 200, "uri": "/legacy"})], + ), + input_schema=LegacyInput.SCHEMA, + output_schema=MockOutput.SCHEMA, + error_registry=TypeRegistry({}), + effective_auth_schemes=[], + error_schemas=[], + ) + input = LegacyInput() + + request = MockBindingProtocol().serialize_request( + operation=operation, + input=input, + endpoint=URI(host="example.com"), + context=TypedProperties(), + ) + + assert input.serialize_called + assert request.method == "POST" + assert request.destination.path == "/legacy" + + +def test_http_binding_protocol_uses_structure_fast_path() -> None: + operation = APIOperation( + input=StructInput, + output=MockOutput, + schema=Schema( + id=ShapeID("ns.foo#StructOperation"), + shape_type=ShapeType.OPERATION, + traits=[HTTPTrait({"method": "POST", "code": 200, "uri": "/structure"})], + ), + input_schema=StructInput.SCHEMA, + output_schema=MockOutput.SCHEMA, + error_registry=TypeRegistry({}), + effective_auth_schemes=[], + error_schemas=[], + ) + input = StructInput() + + request = MockBindingProtocol().serialize_request( + operation=operation, + input=input, + endpoint=URI(host="example.com"), + context=TypedProperties(), + ) + + assert input.serialize_members_called + assert request.method == "POST" + assert request.destination.path == "/structure" + + @pytest.mark.parametrize( "request_uri,endpoint_uri,expected", [ diff --git a/packages/smithy-http/tests/unit/test_bindings.py b/packages/smithy-http/tests/unit/test_bindings.py index f4820dcc2..5b79a80c1 100644 --- a/packages/smithy-http/tests/unit/test_bindings.py +++ b/packages/smithy-http/tests/unit/test_bindings.py @@ -18,6 +18,7 @@ StreamingTrait, ) from smithy_http.bindings import Binding, RequestBindingMatcher, ResponseBindingMatcher +from smithy_http.schema_extensions import HTTP_BINDING_SCHEMA_EXTENSION PAYLOAD_BINDING = Schema.collection( id=ShapeID("com.example#Payload"), @@ -57,7 +58,7 @@ "target": STRING_MAP, "traits": [HTTPQueryParamsTrait()], }, - "header": {"target": STRING, "traits": [HTTPHeaderTrait()]}, + "header": {"target": STRING, "traits": [HTTPHeaderTrait("header")]}, "prefixHeaders": { "target": STRING_MAP, "traits": [HTTPPrefixHeadersTrait("foo")], @@ -156,3 +157,53 @@ def test_response_matching() -> None: assert matcher.match(GENERAL_BINDINGS.members["hostLabel"]) == Binding.BODY assert matcher.match(GENERAL_BINDINGS.members["status"]) == Binding.STATUS assert matcher.match(GENERAL_BINDINGS.members["body"]) == Binding.BODY + + +def test_http_binding_schema_extension_is_cached() -> None: + info = GENERAL_BINDINGS.get_extension(HTTP_BINDING_SCHEMA_EXTENSION) + + assert info is GENERAL_BINDINGS.get_extension(HTTP_BINDING_SCHEMA_EXTENSION) + assert info.request_bindings == tuple( + RequestBindingMatcher(GENERAL_BINDINGS).bindings + ) + assert info.response_bindings == tuple( + ResponseBindingMatcher(GENERAL_BINDINGS).bindings + ) + assert info.has_request_body + assert info.has_response_body + assert info.response_bound_members == ( + ( + GENERAL_BINDINGS.members["header"], + Binding.HEADER, + "header", + False, + ), + ( + GENERAL_BINDINGS.members["prefixHeaders"], + Binding.PREFIX_HEADERS, + "foo", + False, + ), + ( + GENERAL_BINDINGS.members["status"], + Binding.STATUS, + None, + False, + ), + ) + + +def test_http_binding_schema_extension_caches_payload_and_event_stream() -> None: + payload_info = PAYLOAD_BINDING.get_extension(HTTP_BINDING_SCHEMA_EXTENSION) + event_info = EVENT_STREAM_BINDING.get_extension(HTTP_BINDING_SCHEMA_EXTENSION) + + assert payload_info.payload_member is PAYLOAD_BINDING.members["payload"] + assert payload_info.response_bound_members == ( + ( + PAYLOAD_BINDING.members["payload"], + Binding.PAYLOAD, + None, + False, + ), + ) + assert event_info.event_stream_member is EVENT_STREAM_BINDING.members["stream"] diff --git a/packages/smithy-http/tests/unit/test_serializers.py b/packages/smithy-http/tests/unit/test_serializers.py index 1d071d517..64276efb1 100644 --- a/packages/smithy-http/tests/unit/test_serializers.py +++ b/packages/smithy-http/tests/unit/test_serializers.py @@ -23,7 +23,10 @@ TIMESTAMP, ) from smithy_core.schemas import Schema -from smithy_core.serializers import SerializeableShape, ShapeSerializer +from smithy_core.serializers import ( + SerializeableStruct, + ShapeSerializer, +) from smithy_core.shapes import ShapeID, ShapeType from smithy_core.traits import ( EndpointTrait, @@ -36,6 +39,7 @@ HTTPQueryTrait, HTTPResponseCodeTrait, HTTPTrait, + RequiresLengthTrait, StreamingTrait, TimestampFormatTrait, Trait, @@ -43,7 +47,11 @@ from smithy_http import Fields, tuples_to_fields from smithy_http.aio import HTTPResponse as _HTTPResponse from smithy_http.deserializers import HTTPResponseDeserializer -from smithy_http.serializers import HTTPRequestSerializer, HTTPResponseSerializer +from smithy_http.serializers import ( + HTTPBindingSerializer, + HTTPRequestSerializer, + HTTPResponseSerializer, +) from smithy_json import JSONCodec # TODO: empty header prefix, query map @@ -1165,7 +1173,8 @@ class HTTPMessage: status: int = 200 -class Shape(SerializeableShape, DeserializeableShape, Protocol): ... +class Shape(SerializeableStruct, DeserializeableShape, Protocol): + SCHEMA: ClassVar[Schema] @dataclass @@ -1912,16 +1921,16 @@ def async_streaming_payload_cases() -> list[HTTPMessageTestCase]: @pytest.mark.parametrize("case", REQUEST_SER_CASES) async def test_serialize_http_request(case: HTTPMessageTestCase) -> None: - serializer = HTTPRequestSerializer( + serializer = HTTPBindingSerializer( payload_codec=JSONCodec(), + schema=case.shape.SCHEMA, http_trait=case.http_trait, endpoint_trait=case.endpoint_trait, ) - case.shape.serialize(serializer) - actual = serializer.result + case.shape.serialize_members(serializer) + actual = serializer.build_request() expected = case.request - assert actual is not None assert actual.method == expected.method assert actual.destination.host == expected.destination.host assert actual.destination.path == expected.destination.path @@ -1990,6 +1999,38 @@ async def test_serialize_response_omitting_empty_payload() -> None: assert actual_body_value == b"" +async def test_serialize_response_adds_computed_required_content_length() -> None: + schema = Schema.collection( + id=ShapeID("com.smithy#HTTPRequiredLengthPayload"), + members={ + "payload": { + "target": BLOB, + "traits": [ + HTTPPayloadTrait(), + StreamingTrait(), + RequiresLengthTrait(), + ], + } + }, + ) + payload = b"\xde\xad\xbe\xef" + serializer = HTTPResponseSerializer( + payload_codec=JSONCodec(), + http_trait=HTTPTrait({"method": "POST", "code": 200, "uri": "/"}), + ) + + with serializer.begin_struct(schema) as struct_serializer: + struct_serializer.write_data_stream( + schema.members["payload"], + BytesIO(payload), + ) + + actual = serializer.result + assert actual is not None + assert actual.fields["content-length"].as_string() == str(len(payload)) + assert await AsyncBytesReader(actual.body).read() == payload + + RESPONSE_DESER_CASES: list[HTTPMessageTestCase] = ( header_cases() + header_deser_cases() @@ -2018,6 +2059,90 @@ async def test_deserialize_http_response(case: HTTPMessageTestCase) -> None: assert actual == case.shape +def test_deserialize_response_preserves_bound_member_order() -> None: + schema = Schema.collection( + id=ShapeID("com.smithy#OrderedOutput"), + members={ + "status": { + "target": INTEGER, + "traits": [HTTPResponseCodeTrait()], + }, + "header": { + "target": STRING, + "traits": [HTTPHeaderTrait("x-value")], + }, + }, + ) + seen: list[str] = [] + deserializer = HTTPResponseDeserializer( + payload_codec=JSONCodec(), + response=_HTTPResponse( + body=b"", + status=201, + fields=tuples_to_fields([("x-value", "value")]), + ), + body=b"", + ) + + deserializer.read_struct( + schema, + lambda member, _: seen.append(member.expect_member_name()), + ) + + assert seen == ["status", "header"] + + +def test_deserialize_recursive_response_uses_original_schema() -> None: + recursive_schema = Schema.collection( + id=ShapeID("com.smithy#RecursiveOutput"), + members={ + "header": { + "target": STRING, + "traits": [HTTPHeaderTrait("x-value")], + }, + "child": None, + }, + ) + recursive_schema.members["child"] = Schema.member( + id=recursive_schema.id.with_member("child"), + target=recursive_schema, + index=1, + ) + + def deserialize_shape(deserializer: ShapeDeserializer) -> dict[str, Any]: + result: dict[str, Any] = {} + + def consume(member: Schema, member_deserializer: ShapeDeserializer) -> None: + match member.expect_member_index(): + case 0: + result["header"] = member_deserializer.read_string( + recursive_schema.members["header"] + ) + case 1: + result["child"] = deserialize_shape(member_deserializer) + case _: + raise AssertionError(f"Unexpected member: {member.id}") + + deserializer.read_struct(recursive_schema, consume) + return result + + body = b'{"child":{"header":"nested"}}' + deserializer = HTTPResponseDeserializer( + payload_codec=JSONCodec(), + response=_HTTPResponse( + body=body, + status=200, + fields=tuples_to_fields([("x-value", "top")]), + ), + body=body, + ) + + assert deserialize_shape(deserializer) == { + "header": "top", + "child": {"header": "nested"}, + } + + async def test_deserialize_http_response_with_async_stream() -> None: stream = AsyncBytesReader(b"\xde\xad\xbe\xef")