From 7d6de63cc4f1e1bd93dba457f9582b4e237b93ee Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 18 Jul 2026 23:00:40 -0400 Subject: [PATCH 01/10] Document Python code generator design --- designs/codegen/cli.md | 95 ++++++++++++++++++++++++++++++++++++++++ designs/codegen/index.md | 60 +++++++++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 designs/codegen/cli.md create mode 100644 designs/codegen/index.md diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md new file mode 100644 index 000000000..dec4395fa --- /dev/null +++ b/designs/codegen/cli.md @@ -0,0 +1,95 @@ +# Code Generator CLI + +The `smithy-python` command is the process interface described in the +[Python Code Generation](index.md) overview. It supports direct use and +invocation from Smithy's +[`run` plugin](https://smithy.io/2.0/guides/smithy-build-json.html#run-plugin). + +## Commands + +Generation is organized by artifact type: + +```console +smithy-python generate client [OPTIONS] +smithy-python generate types [OPTIONS] +``` + +`client` generates a service client and its required types. `types` generates a +standalone types package. Both commands accept the following process options: + +* `--model PATH` reads a JSON AST from a file instead of standard input. +* `--output PATH` selects the output directory for direct invocation. + +Settings specific to each artifact will be added with the functionality that +consumes them. + +The command MUST return zero after successful generation and non-zero when +arguments, settings, the model, or generation are invalid. Diagnostics are +written to standard error. Invalid command syntax and invocation inputs return +2, while I/O and generation failures return 1. + +## Smithy `run` Plugin + +The Smithy `run` plugin executes an external program during a build. It sends the +projection's Smithy model as a JSON AST to the process's standard input and runs +the process in the plugin's output directory. + +A plugin ID MUST use `run::` followed by a custom artifact name. The configured +command identifies the artifact to generate: + +```json +{ + "version": "1.0", + "projections": { + "client": { + "plugins": { + "run::python-client": { + "command": ["smithy-python", "generate", "client"] + } + } + } + } +} +``` + +Artifact-specific options will be appended to `command` after they are defined. + +The `smithy-python` executable MUST be installed or otherwise available on the +Smithy process's `PATH`. Smithy passes no arguments other than those in +`command`. + +### Input and Output + +When invoked by Smithy, the CLI reads one JSON AST document from standard input. +The document represents the model after projection transforms have been applied. + +The presence of `SMITHY_PLUGIN_DIR` identifies an invocation by the `run` plugin. +Generated files are written beneath this directory, which Smithy also uses as the +process's working directory. The CLI MUST NOT write generated files outside it, +and `--model` and `--output` MUST NOT be used in this mode. + +The `run` plugin provides the following environment variables: + +| Name | Purpose | +|------|---------| +| `SMITHY_ROOT_DIR` | Root directory of the Smithy build. | +| `SMITHY_PLUGIN_DIR` | Output and working directory for the plugin. | +| `SMITHY_PROJECTION_NAME` | Name of the active projection. | +| `SMITHY_ARTIFACT_NAME` | Custom artifact name from the plugin ID. | +| `SMITHY_INCLUDES_PRELUDE` | Whether the JSON AST includes prelude shapes. | + +The CLI uses this context to interpret the model. Protocol and platform +integrations MAY also use it while generating files. + +Smithy omits prelude shapes by default. A build MAY set `sendPrelude` to `true` +in the `run` plugin configuration when those shapes are needed. + +## Direct Invocation + +When `SMITHY_PLUGIN_DIR` is absent, the CLI treats the command as a direct +invocation and requires `--output`. It follows the same +generation path as Smithy invocation and can read a JSON AST from a file instead +of standard input by using `--model`. When standard input is an interactive +terminal, `--model` is required so that an omitted input does not wait indefinitely +for input. This mode is intended for development, testing, and integration with +tools other than the Smithy CLI. diff --git a/designs/codegen/index.md b/designs/codegen/index.md new file mode 100644 index 000000000..c6924759d --- /dev/null +++ b/designs/codegen/index.md @@ -0,0 +1,60 @@ +# Python Code Generation + +Smithy Python currently generates clients with the Java implementation in +`codegen`. This document describes the Python code generator that will replace +that implementation over time. + +The Python generator is distributed as `smithy-python`. It is separate from the +runtime packages used by generated code, and is only needed while generating a +package. + +## Goals + +* Generate Python clients and standalone types packages from Smithy models. +* Integrate with standard Smithy builds without requiring a Java code generator. +* Provide extension points for protocol and platform-specific behavior. +* Produce code compatible with the existing Smithy Python runtime packages. +* Allow the Python and Java generators to coexist during migration. + +## Architecture + +The generator consumes a Smithy JSON AST and settings for an artifact. It loads +the model, applies artifact and protocol-specific behavior, and writes a Python +package. + +```text +Smithy JSON AST + settings + | + v + smithy-python generator + | + v + client or types package +``` + +Two artifact types are initially planned: + +* `client` will generate a service client and its required types. +* `types` will generate a standalone package of types selected from a model. + +The command-line interface is the generator's first entry point. Smithy's `run` +plugin invokes it as an external process, so the generator does not need to be +loaded into the Smithy CLI or implemented in Java. + +Generated packages MUST NOT depend on `smithy-python` at runtime. They MAY +depend on the handwritten runtime packages in this repository. + +## Migration + +The Java generator remains authoritative while the Python generator is under +development. Features may be implemented and reviewed incrementally without +changing the Java path. A generated artifact SHOULD move to the Python generator +only after the required behavior is supported and tested. + +The Python generator does not need to reproduce Java implementation details or +byte-for-byte output. It MUST preserve the supported Smithy semantics and public +behavior of generated packages. + +## Designs + +* [Code Generator CLI](cli.md) From bc7ff3d6153a69f7aefc916c4ca5e602a6a76a46 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 18 Jul 2026 23:01:30 -0400 Subject: [PATCH 02/10] Add experimental Python codegen CLI --- README.md | 11 +- .../smithy-python-feature-codegen-cli.json | 4 + packages/smithy-python/CHANGELOG.md | 1 + packages/smithy-python/NOTICE | 1 + packages/smithy-python/README.md | 19 ++ packages/smithy-python/pyproject.toml | 51 ++++ .../src/smithy_python/__init__.py | 5 + .../src/smithy_python/__main__.py | 7 + .../smithy-python/src/smithy_python/cli.py | 140 +++++++++++ .../src/smithy_python/environment.py | 40 ++++ .../src/smithy_python/exceptions.py | 15 ++ .../smithy-python/src/smithy_python/py.typed | 0 packages/smithy-python/tests/unit/__init__.py | 2 + packages/smithy-python/tests/unit/test_cli.py | 225 ++++++++++++++++++ .../tests/unit/test_environment.py | 43 ++++ .../tests/unit/test_exceptions.py | 14 ++ pyproject.toml | 3 +- uv.lock | 5 + 18 files changed, 582 insertions(+), 4 deletions(-) create mode 100644 packages/smithy-python/.changes/next-release/smithy-python-feature-codegen-cli.json create mode 100644 packages/smithy-python/CHANGELOG.md create mode 100644 packages/smithy-python/NOTICE create mode 100644 packages/smithy-python/README.md create mode 100644 packages/smithy-python/pyproject.toml create mode 100644 packages/smithy-python/src/smithy_python/__init__.py create mode 100644 packages/smithy-python/src/smithy_python/__main__.py create mode 100644 packages/smithy-python/src/smithy_python/cli.py create mode 100644 packages/smithy-python/src/smithy_python/environment.py create mode 100644 packages/smithy-python/src/smithy_python/exceptions.py create mode 100644 packages/smithy-python/src/smithy_python/py.typed create mode 100644 packages/smithy-python/tests/unit/__init__.py create mode 100644 packages/smithy-python/tests/unit/test_cli.py create mode 100644 packages/smithy-python/tests/unit/test_environment.py create mode 100644 packages/smithy-python/tests/unit/test_exceptions.py diff --git a/README.md b/README.md index cbda5db4c..1d32c4d75 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,11 @@ This code generator, and the clients it generates, are unstable and should not be used in production systems yet. Several features, such as detailed logging, have not been implemented yet. +> [!NOTE] +> The Java generator in `codegen` remains the authoritative implementation. +> `packages/smithy-python` contains an experimental Python-native CLI scaffold +> that does not generate code yet. + ### What is this repository? This repository contains two major components: @@ -20,9 +25,9 @@ This repository contains two major components: 2) Core modules and interfaces for building service clients in Python These components facilitate generating clients for any [Smithy](https://smithy.io/) -service. The `codegen` directory contains the source code for generating clients. -The `python-packages` directory contains the source code for the handwritten python -components. +service. The `codegen` directory contains the current Java generator, +`packages/smithy-python` contains the Python-native generator scaffold, and the +other directories under `packages` contain the handwritten Python components. This repository does *not* contain any generated clients, such as for S3 or other AWS services. Rather, these are the tools that facilitate the generation of those diff --git a/packages/smithy-python/.changes/next-release/smithy-python-feature-codegen-cli.json b/packages/smithy-python/.changes/next-release/smithy-python-feature-codegen-cli.json new file mode 100644 index 000000000..e9ed269f8 --- /dev/null +++ b/packages/smithy-python/.changes/next-release/smithy-python-feature-codegen-cli.json @@ -0,0 +1,4 @@ +{ + "type": "feature", + "description": "Added the experimental smithy-python package and CLI scaffold." +} diff --git a/packages/smithy-python/CHANGELOG.md b/packages/smithy-python/CHANGELOG.md new file mode 100644 index 000000000..825c32f0d --- /dev/null +++ b/packages/smithy-python/CHANGELOG.md @@ -0,0 +1 @@ +# Changelog diff --git a/packages/smithy-python/NOTICE b/packages/smithy-python/NOTICE new file mode 100644 index 000000000..616fc5889 --- /dev/null +++ b/packages/smithy-python/NOTICE @@ -0,0 +1 @@ +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/packages/smithy-python/README.md b/packages/smithy-python/README.md new file mode 100644 index 000000000..9154bc542 --- /dev/null +++ b/packages/smithy-python/README.md @@ -0,0 +1,19 @@ +# smithy-python + +> [!WARNING] +> This package is an experimental scaffold. It does not generate code yet. The +> Java generator in the repository's `codegen` directory remains authoritative. + +`smithy-python` will provide Python-native code generation for Smithy models. +The initial command-line interface exposes the planned client and types generation +commands so that their top-level shape can be developed independently from the +generator implementation. + +```console +smithy-python generate client +smithy-python generate types +``` + +Both generation commands currently exit with an error explaining that generation +has not been implemented. The package is included in workspace builds to validate +its packaging and entry points. diff --git a/packages/smithy-python/pyproject.toml b/packages/smithy-python/pyproject.toml new file mode 100644 index 000000000..3bf5dccd0 --- /dev/null +++ b/packages/smithy-python/pyproject.toml @@ -0,0 +1,51 @@ +[project] +name = "smithy-python" +dynamic = ["version"] +requires-python = ">=3.12" +authors = [ + {name = "Amazon Web Services"}, +] +description = "A Smithy code generator for Python clients and types." +readme = "README.md" +license = {text = "Apache License 2.0"} +keywords = ["smithy", "codegen", "sdk"] +classifiers = [ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Developers", + "Natural Language :: English", + "License :: OSI Approved :: Apache Software License", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3 :: Only", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Free Threading :: 2 - Beta", + "Topic :: Software Development :: Code Generators", +] +dependencies = [] + +[project.scripts] +smithy-python = "smithy_python.cli:main" + +[project.urls] +"Changelog" = "https://github.com/smithy-lang/smithy-python/blob/develop/packages/smithy-python/CHANGELOG.md" +"Code" = "https://github.com/smithy-lang/smithy-python/tree/develop/packages/smithy-python/" +"Issue tracker" = "https://github.com/smithy-lang/smithy-python/issues" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.version] +path = "src/smithy_python/__init__.py" + +[tool.hatch.build] +exclude = [ + "tests", +] + +[tool.ruff] +src = ["src"] diff --git a/packages/smithy-python/src/smithy_python/__init__.py b/packages/smithy-python/src/smithy_python/__init__.py new file mode 100644 index 000000000..c3621cb0c --- /dev/null +++ b/packages/smithy-python/src/smithy_python/__init__.py @@ -0,0 +1,5 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A Smithy code generator for Python clients and types.""" + +__version__ = "0.0.0" diff --git a/packages/smithy-python/src/smithy_python/__main__.py b/packages/smithy-python/src/smithy_python/__main__.py new file mode 100644 index 000000000..b8aaf8e60 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/__main__.py @@ -0,0 +1,7 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from .cli import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py new file mode 100644 index 000000000..7736312c8 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -0,0 +1,140 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Command-line interface for the Smithy Python code generator.""" + +from __future__ import annotations + +import argparse +import os +import sys +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import BinaryIO, Final + +from . import __version__ +from .environment import PluginEnvironment +from .exceptions import CodegenError, InvalidInvocationError + +_GENERATION_NOT_IMPLEMENTED: Final = ( + "smithy-python: error: {artifact} generation is not implemented yet\n" +) + + +@dataclass(frozen=True, slots=True) +class _Invocation: + artifact: str + model_source: bytes + output_dir: Path + environment: PluginEnvironment + + +def main( + argv: Sequence[str] | None = None, + *, + environ: Mapping[str, str] | None = None, + stdin: BinaryIO | None = None, +) -> int: + """Run the CLI with the provided process inputs and return its exit code.""" + parser = _create_parser() + + try: + args = parser.parse_args(argv) + except SystemExit as error: + return error.code if isinstance(error.code, int) else 1 + + try: + _resolve_invocation( + args, + environ=os.environ if environ is None else environ, + stdin=stdin, + ) + except InvalidInvocationError as error: + sys.stderr.write(f"smithy-python: error: {error}\n") + return 2 + except (CodegenError, OSError) as error: + sys.stderr.write(f"smithy-python: error: {error}\n") + return 1 + + sys.stderr.write(_GENERATION_NOT_IMPLEMENTED.format(artifact=args.artifact)) + return 1 + + +def _create_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="smithy-python", + description="Generate Python source from Smithy models.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {__version__}", + ) + + commands = parser.add_subparsers(required=True) + generate = commands.add_parser("generate", help="Generate Python source") + artifacts = generate.add_subparsers(dest="artifact", required=True) + for name, help_text in ( + ("client", "Generate a client package"), + ("types", "Generate a standalone types package"), + ): + artifact = artifacts.add_parser(name, help=help_text) + artifact.add_argument( + "--model", + type=Path, + help="Read the JSON AST from a file instead of standard input", + ) + artifact.add_argument( + "--output", + type=Path, + help="Output directory for direct invocation", + ) + + return parser + + +def _resolve_invocation( + args: argparse.Namespace, + *, + environ: Mapping[str, str], + stdin: BinaryIO | None, +) -> _Invocation: + environment = PluginEnvironment.from_environ(environ) + model_path: Path | None = args.model + output_path: Path | None = args.output + + if (plugin_dir := environment.plugin_dir) is not None: + if model_path is not None: + raise InvalidInvocationError( + "--model cannot be used with the Smithy run plugin" + ) + if output_path is not None: + raise InvalidInvocationError( + "--output cannot be used with the Smithy run plugin" + ) + output_dir = plugin_dir + else: + if output_path is None: + raise InvalidInvocationError("Direct invocation requires --output") + output_dir = output_path + + if model_path is not None: + if not model_path.is_file(): + raise InvalidInvocationError(f"Model path is not a file: {model_path}") + model_source = model_path.read_bytes() + else: + model_stream = sys.stdin.buffer if stdin is None else stdin + if environment.plugin_dir is None and model_stream.isatty(): + raise InvalidInvocationError( + "Direct invocation requires --model or a model piped to standard input" + ) + model_source = model_stream.read() + if not model_source: + raise InvalidInvocationError("Expected a Smithy JSON AST model") + + return _Invocation( + artifact=args.artifact, + model_source=model_source, + output_dir=output_dir, + environment=environment, + ) diff --git a/packages/smithy-python/src/smithy_python/environment.py b/packages/smithy-python/src/smithy_python/environment.py new file mode 100644 index 000000000..5137519ae --- /dev/null +++ b/packages/smithy-python/src/smithy_python/environment.py @@ -0,0 +1,40 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Smithy build environment provided to the code generator.""" + +from __future__ import annotations + +import os +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Self + + +@dataclass(frozen=True, slots=True) +class PluginEnvironment: + """Environment values supplied by Smithy's process-based run plugin.""" + + root_dir: Path | None = None + plugin_dir: Path | None = None + projection_name: str | None = None + artifact_name: str | None = None + includes_prelude: bool = False + + @classmethod + def from_environ(cls, environ: Mapping[str, str] | None = None) -> Self: + """Load the Smithy run plugin environment from a mapping.""" + + source = os.environ if environ is None else environ + + def path(name: str) -> Path | None: + return Path(value) if (value := source.get(name)) else None + + return cls( + root_dir=path("SMITHY_ROOT_DIR"), + plugin_dir=path("SMITHY_PLUGIN_DIR"), + projection_name=source.get("SMITHY_PROJECTION_NAME"), + artifact_name=source.get("SMITHY_ARTIFACT_NAME"), + includes_prelude=source.get("SMITHY_INCLUDES_PRELUDE", "false").lower() + == "true", + ) diff --git a/packages/smithy-python/src/smithy_python/exceptions.py b/packages/smithy-python/src/smithy_python/exceptions.py new file mode 100644 index 000000000..a0cff3f93 --- /dev/null +++ b/packages/smithy-python/src/smithy_python/exceptions.py @@ -0,0 +1,15 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exceptions raised by Smithy Python code generation.""" + + +class SmithyPythonError(Exception): + """Base exception for errors raised by the Smithy Python generator.""" + + +class CodegenError(SmithyPythonError): + """Raised when code generation fails.""" + + +class InvalidInvocationError(SmithyPythonError): + """Raised when command-line inputs do not form a valid invocation.""" diff --git a/packages/smithy-python/src/smithy_python/py.typed b/packages/smithy-python/src/smithy_python/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/packages/smithy-python/tests/unit/__init__.py b/packages/smithy-python/tests/unit/__init__.py new file mode 100644 index 000000000..04f8b7b76 --- /dev/null +++ b/packages/smithy-python/tests/unit/__init__.py @@ -0,0 +1,2 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py new file mode 100644 index 000000000..a82c87a1a --- /dev/null +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -0,0 +1,225 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import importlib +from io import BytesIO +from pathlib import Path + +import pytest +from smithy_python import __version__ +from smithy_python.cli import main + + +class _InteractiveStdin(BytesIO): + def isatty(self) -> bool: + return True + + +@pytest.mark.parametrize( + ("argv", "expected"), + [ + (("--help",), "usage: smithy-python"), + (("--version",), f"smithy-python {__version__}"), + ], +) +def test_information_commands( + argv: tuple[str, ...], expected: str, capsys: pytest.CaptureFixture[str] +) -> None: + assert main(argv) == 0 + assert capsys.readouterr().out.startswith(expected) + + +@pytest.mark.parametrize("artifact", ["client", "types"]) +def test_generation_commands_are_explicitly_unavailable( + artifact: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + model = tmp_path / "model.json" + model.write_text("{}") + + assert ( + main( + ( + "generate", + artifact, + "--model", + str(model), + "--output", + str(tmp_path / "output"), + ), + environ={}, + ) + == 1 + ) + assert capsys.readouterr().err == ( + f"smithy-python: error: {artifact} generation is not implemented yet\n" + ) + + +@pytest.mark.parametrize( + ("argv", "expected_usage"), + [ + ((), "usage: smithy-python"), + (("generate",), "usage: smithy-python generate"), + ], +) +def test_missing_command_identifies_available_subcommands( + argv: tuple[str, ...], + expected_usage: str, + capsys: pytest.CaptureFixture[str], +) -> None: + assert main(argv) == 2 + assert capsys.readouterr().err.startswith(expected_usage) + + +def test_main_module_can_be_imported() -> None: + importlib.import_module("smithy_python.__main__") + + +def test_run_plugin_invocation_reads_standard_input( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert ( + main( + ("generate", "client"), + environ={"SMITHY_PLUGIN_DIR": str(tmp_path)}, + stdin=BytesIO(b"{}"), + ) + == 1 + ) + assert "generation is not implemented yet" in capsys.readouterr().err + + +@pytest.mark.parametrize("option", ["--model", "--output"]) +def test_run_plugin_rejects_direct_invocation_options( + option: str, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + assert ( + main( + ("generate", "client", option, str(tmp_path / "value")), + environ={"SMITHY_PLUGIN_DIR": str(tmp_path)}, + stdin=BytesIO(b"{}"), + ) + == 2 + ) + assert f"{option} cannot be used with the Smithy run plugin" in ( + capsys.readouterr().err + ) + + +def test_direct_invocation_requires_output( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + model = tmp_path / "model.json" + model.write_text("{}") + + assert main(("generate", "client", "--model", str(model)), environ={}) == 2 + assert "Direct invocation requires --output" in capsys.readouterr().err + + +def test_invocation_rejects_empty_model( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert ( + main( + ("generate", "client", "--output", str(tmp_path)), + environ={}, + stdin=BytesIO(), + ) + == 2 + ) + assert "Expected a Smithy JSON AST model" in capsys.readouterr().err + + +def test_direct_invocation_rejects_interactive_model_input( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert ( + main( + ("generate", "client", "--output", str(tmp_path)), + environ={}, + stdin=_InteractiveStdin(), + ) + == 2 + ) + assert ( + "Direct invocation requires --model or a model piped to standard input" + in capsys.readouterr().err + ) + + +def test_invocation_reports_unreadable_model( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + missing = tmp_path / "missing.json" + + assert ( + main( + ( + "generate", + "client", + "--model", + str(missing), + "--output", + str(tmp_path), + ), + environ={}, + ) + == 2 + ) + assert f"Model path is not a file: {missing}" in capsys.readouterr().err + + +def test_invocation_rejects_empty_model_path( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + assert ( + main( + ( + "generate", + "client", + "--model", + "", + "--output", + str(tmp_path), + ), + environ={}, + ) + == 2 + ) + assert "Model path is not a file: ." in capsys.readouterr().err + + +def test_invocation_reports_model_io_error( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + model = tmp_path / "model.json" + model.write_text("{}") + + def raise_io_error(self: Path) -> bytes: + raise OSError("unable to read model") + + monkeypatch.setattr(Path, "read_bytes", raise_io_error) + + assert ( + main( + ( + "generate", + "client", + "--model", + str(model), + "--output", + str(tmp_path), + ), + environ={}, + ) + == 1 + ) + assert "unable to read model" in capsys.readouterr().err diff --git a/packages/smithy-python/tests/unit/test_environment.py b/packages/smithy-python/tests/unit/test_environment.py new file mode 100644 index 000000000..ea8e270fd --- /dev/null +++ b/packages/smithy-python/tests/unit/test_environment.py @@ -0,0 +1,43 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from pathlib import Path + +import pytest +from smithy_python.environment import PluginEnvironment + + +def test_loads_run_plugin_environment() -> None: + environment = PluginEnvironment.from_environ( + { + "SMITHY_ROOT_DIR": "/tmp/root", + "SMITHY_PLUGIN_DIR": "/tmp/plugin", + "SMITHY_PROJECTION_NAME": "client", + "SMITHY_ARTIFACT_NAME": "python-client", + "SMITHY_INCLUDES_PRELUDE": "true", + } + ) + + assert environment.root_dir == Path("/tmp/root") + assert environment.plugin_dir == Path("/tmp/plugin") + assert environment.projection_name == "client" + assert environment.artifact_name == "python-client" + assert environment.includes_prelude + + +def test_defaults_to_direct_invocation() -> None: + environment = PluginEnvironment.from_environ({}) + + assert environment.root_dir is None + assert environment.plugin_dir is None + assert environment.projection_name is None + assert environment.artifact_name is None + assert not environment.includes_prelude + + +def test_loads_os_environment_by_default( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("SMITHY_PLUGIN_DIR", str(tmp_path)) + + assert PluginEnvironment.from_environ().plugin_dir == tmp_path diff --git a/packages/smithy-python/tests/unit/test_exceptions.py b/packages/smithy-python/tests/unit/test_exceptions.py new file mode 100644 index 000000000..bd1867f56 --- /dev/null +++ b/packages/smithy-python/tests/unit/test_exceptions.py @@ -0,0 +1,14 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 + +from smithy_python.exceptions import ( + CodegenError, + InvalidInvocationError, + SmithyPythonError, +) + + +def test_error_hierarchy_distinguishes_invocation_and_codegen_failures() -> None: + assert issubclass(CodegenError, SmithyPythonError) + assert issubclass(InvalidInvocationError, SmithyPythonError) + assert not issubclass(InvalidInvocationError, CodegenError) diff --git a/pyproject.toml b/pyproject.toml index 152cf5f61..42a463e81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [project] -name = "smithy-python" +name = "smithy-python-workspace" version = "0.1.0" description = "Add your description here" readme = "README.md" @@ -39,6 +39,7 @@ smithy_xml = { workspace = true } smithy_aws_core = { workspace = true } smithy_aws_event_stream = { workspace = true } aws_sdk_signers = {workspace = true } +smithy_python = { workspace = true } [tool.pyright] typeCheckingMode = "strict" diff --git a/uv.lock b/uv.lock index f55a3a435..59d64a67c 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,7 @@ members = [ "smithy-http", "smithy-json", "smithy-python", + "smithy-python-workspace", "smithy-xml", ] @@ -776,6 +777,10 @@ requires-dist = [ [[package]] name = "smithy-python" +source = { editable = "packages/smithy-python" } + +[[package]] +name = "smithy-python-workspace" version = "0.1.0" source = { virtual = "." } From cbe477ccfaa274d1d69fa7f67aa5f7c6b3f7f8c9 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 18 Jul 2026 23:21:24 -0400 Subject: [PATCH 03/10] Remove redundant exception hierarchy test --- .../smithy-python/tests/unit/test_exceptions.py | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 packages/smithy-python/tests/unit/test_exceptions.py diff --git a/packages/smithy-python/tests/unit/test_exceptions.py b/packages/smithy-python/tests/unit/test_exceptions.py deleted file mode 100644 index bd1867f56..000000000 --- a/packages/smithy-python/tests/unit/test_exceptions.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 - -from smithy_python.exceptions import ( - CodegenError, - InvalidInvocationError, - SmithyPythonError, -) - - -def test_error_hierarchy_distinguishes_invocation_and_codegen_failures() -> None: - assert issubclass(CodegenError, SmithyPythonError) - assert issubclass(InvalidInvocationError, SmithyPythonError) - assert not issubclass(InvalidInvocationError, CodegenError) From 0f09ecee9276c20c209327fdef36df44fa18b451 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 19 Jul 2026 17:23:54 -0400 Subject: [PATCH 04/10] Clarify CLI examples and test module entry point Mark generation command examples as schematic and clarify option validation behavior. Exercise python -m smithy_python in a subprocess. --- packages/smithy-python/README.md | 10 +++++----- packages/smithy-python/tests/unit/test_cli.py | 16 +++++++++++++--- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/packages/smithy-python/README.md b/packages/smithy-python/README.md index 9154bc542..79db2d4bc 100644 --- a/packages/smithy-python/README.md +++ b/packages/smithy-python/README.md @@ -10,10 +10,10 @@ commands so that their top-level shape can be developed independently from the generator implementation. ```console -smithy-python generate client -smithy-python generate types +smithy-python generate client [OPTIONS] +smithy-python generate types [OPTIONS] ``` -Both generation commands currently exit with an error explaining that generation -has not been implemented. The package is included in workspace builds to validate -its packaging and entry points. +After validating their invocation options, both generation commands currently exit +with an error explaining that generation has not been implemented. The package is +included in workspace builds to validate its packaging and entry points. diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py index a82c87a1a..74a2418cb 100644 --- a/packages/smithy-python/tests/unit/test_cli.py +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -3,7 +3,8 @@ from __future__ import annotations -import importlib +import subprocess +import sys from io import BytesIO from pathlib import Path @@ -75,8 +76,17 @@ def test_missing_command_identifies_available_subcommands( assert capsys.readouterr().err.startswith(expected_usage) -def test_main_module_can_be_imported() -> None: - importlib.import_module("smithy_python.__main__") +def test_main_module_invokes_cli() -> None: + result = subprocess.run( + [sys.executable, "-m", "smithy_python", "--version"], + capture_output=True, + check=False, + text=True, + ) + + assert result.returncode == 0 + assert result.stdout == f"smithy-python {__version__}\n" + assert result.stderr == "" def test_run_plugin_invocation_reads_standard_input( From ba91974bc9ff273173dd44328d22bba1f88ed453 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 21 Jul 2026 23:48:01 -0400 Subject: [PATCH 05/10] Address PR feedback --- designs/codegen/cli.md | 3 +- designs/codegen/index.md | 3 ++ .../smithy-python/src/smithy_python/cli.py | 35 +++++++++++++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md index dec4395fa..99562194f 100644 --- a/designs/codegen/cli.md +++ b/designs/codegen/cli.md @@ -18,7 +18,8 @@ smithy-python generate types [OPTIONS] standalone types package. Both commands accept the following process options: * `--model PATH` reads a JSON AST from a file instead of standard input. -* `--output PATH` selects the output directory for direct invocation. +* `--output PATH` selects the output directory. It defaults to the Smithy run + plugin's output directory (`SMITHY_PLUGIN_DIR`) when invoked by Smithy. Settings specific to each artifact will be added with the functionality that consumes them. diff --git a/designs/codegen/index.md b/designs/codegen/index.md index c6924759d..9901b8be1 100644 --- a/designs/codegen/index.md +++ b/designs/codegen/index.md @@ -37,6 +37,9 @@ Two artifact types are initially planned: * `client` will generate a service client and its required types. * `types` will generate a standalone package of types selected from a model. +The artifact set may grow over time. A `server` artifact is a natural addition, +so the generator should not assume that only `client` and `types` exist. + The command-line interface is the generator's first entry point. Smithy's `run` plugin invokes it as an external process, so the generator does not need to be loaded into the Smithy CLI or implemented in Java. diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index 7736312c8..79cf26a62 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -74,22 +74,35 @@ def _create_parser() -> argparse.ArgumentParser: commands = parser.add_subparsers(required=True) generate = commands.add_parser("generate", help="Generate Python source") artifacts = generate.add_subparsers(dest="artifact", required=True) + common = _common_artifact_options() for name, help_text in ( ("client", "Generate a client package"), ("types", "Generate a standalone types package"), ): - artifact = artifacts.add_parser(name, help=help_text) - artifact.add_argument( - "--model", - type=Path, - help="Read the JSON AST from a file instead of standard input", - ) - artifact.add_argument( - "--output", - type=Path, - help="Output directory for direct invocation", - ) + artifacts.add_parser(name, help=help_text, parents=[common]) + + return parser + +def _common_artifact_options() -> argparse.ArgumentParser: + """Build a parent parser with the options shared by every artifact.""" + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument( + "--model", + type=Path, + help=( + "The Smithy JSON AST model file to use for code generation. " + "If not set, the model is read from standard input." + ), + ) + parser.add_argument( + "--output", + type=Path, + help=( + "Output directory for generated files. Defaults to the Smithy run " + "plugin's output directory (SMITHY_PLUGIN_DIR) when invoked by Smithy." + ), + ) return parser From 05ea279dee6109056e258abc2acfe80663a024f4 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 12 Sep 2026 01:40:16 -0400 Subject: [PATCH 06/10] docs(codegen): Clarify service selection and generated shapes Document that --service is optional when the model contains a single service, that both artifacts generate every data shape in the model rather than the service closure, and that case-insensitive name conflicts are a hard error. Note that run plugin env settings may back command-line options. --- designs/codegen/cli.md | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md index 99562194f..241436dbc 100644 --- a/designs/codegen/cli.md +++ b/designs/codegen/cli.md @@ -14,8 +14,9 @@ smithy-python generate client [OPTIONS] smithy-python generate types [OPTIONS] ``` -`client` generates a service client and its required types. `types` generates a -standalone types package. Both commands accept the following process options: +`client` generates a service client together with the data shapes in the model. +`types` generates a standalone package containing only the data shapes. Both +commands accept the following process options: * `--model PATH` reads a JSON AST from a file instead of standard input. * `--output PATH` selects the output directory. It defaults to the Smithy run @@ -24,6 +25,33 @@ standalone types package. Both commands accept the following process options: Settings specific to each artifact will be added with the functionality that consumes them. +### Service Selection + +The CLI does not require a service to be named. It resolves the service to +generate as follows: + +* `--service SHAPE_ID` selects a specific service shape. The shape MUST exist in + the model and MUST be a service. +* When `--service` is omitted and the model contains exactly one service shape, + that service is used. +* When `--service` is omitted and the model contains more than one service + shape, the command fails with an invocation error that lists the candidates. + +The `client` artifact requires a resolved service. The `types` artifact does +not. The CLI MUST NOT synthesize a placeholder service to satisfy generation. + +### Generated Shapes + +Both artifacts generate every data shape in the model they receive; the set is +not narrowed to the closure of the selected service. Builds that want a smaller +package apply smithy-build transforms in the projection. Trait definitions, +prelude shapes, and shapes marked `@private` or `@mixin` are never generated. + +Because the model is not limited to a service closure, shape names are not +guaranteed to be unique. When two generated shapes have case-insensitively equal +names, the command fails with an error that identifies the conflicting shape +IDs. + The command MUST return zero after successful generation and non-zero when arguments, settings, the model, or generation are invalid. Diagnostics are written to standard error. Invalid command syntax and invocation inputs return @@ -54,6 +82,9 @@ command identifies the artifact to generate: ``` Artifact-specific options will be appended to `command` after they are defined. +The `run` plugin can also pass settings through its `env` property, so an option +MAY additionally be read from an environment variable. A command-line option +takes precedence over its environment variable. The `smithy-python` executable MUST be installed or otherwise available on the Smithy process's `PATH`. Smithy passes no arguments other than those in From b020d91a7f94ee321dbae34e8d59c095a0847107 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 12 Sep 2026 22:08:01 -0400 Subject: [PATCH 07/10] docs(codegen): Generate the service closure by default Generating every shape in the model diverged from every other Smithy generator and fails on a published AWS model whose leaked, unconnected shapes collide with real ones. Document the service closure as the default selection, with a note when shapes are left out, and keep the whole-model behavior for the types artifact when no service is present. --- designs/codegen/cli.md | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md index 241436dbc..c01fb8f7c 100644 --- a/designs/codegen/cli.md +++ b/designs/codegen/cli.md @@ -14,9 +14,9 @@ smithy-python generate client [OPTIONS] smithy-python generate types [OPTIONS] ``` -`client` generates a service client together with the data shapes in the model. -`types` generates a standalone package containing only the data shapes. Both -commands accept the following process options: +`client` generates a service client and the data shapes it uses. `types` +generates a standalone package containing only data shapes. Both commands accept +the following process options: * `--model PATH` reads a JSON AST from a file instead of standard input. * `--output PATH` selects the output directory. It defaults to the Smithy run @@ -42,15 +42,22 @@ not. The CLI MUST NOT synthesize a placeholder service to satisfy generation. ### Generated Shapes -Both artifacts generate every data shape in the model they receive; the set is -not narrowed to the closure of the selected service. Builds that want a smaller -package apply smithy-build transforms in the projection. Trait definitions, -prelude shapes, and shapes marked `@private` or `@mixin` are never generated. - -Because the model is not limited to a service closure, shape names are not -guaranteed to be unique. When two generated shapes have case-insensitively equal -names, the command fails with an error that identifies the conflicting shape -IDs. +When a service is resolved, both artifacts generate the data shapes in the +service closure: every shape reachable from the service through its operations, +resources, errors, and members. This matches the surface produced by the other +Smithy code generators. Data shapes in the model that are not connected to the +service are not generated, and the CLI reports how many were left out. + +When no service is resolved, the `types` artifact generates every data shape in +the model. Smithy guarantees case-insensitively unique shape names only within a +service closure, so in this mode the command fails when two shapes have +case-insensitively equal names, identifying the conflicting shape IDs. + +Trait definitions, prelude shapes, and shapes marked `@mixin` are never +generated. Builds that need a different set of shapes, such as types that are +not bound to any operation, apply smithy-build transforms in the projection. +An option to generate every shape in the model regardless of the service MAY be +added when there is a need for it. The command MUST return zero after successful generation and non-zero when arguments, settings, the model, or generation are invalid. Diagnostics are From e6b30266a92a4797318e1030ff1542b6c42138e0 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sat, 12 Sep 2026 22:23:53 -0400 Subject: [PATCH 08/10] docs(codegen): Document mixin resolution during model loading --- designs/codegen/cli.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md index c01fb8f7c..d84f5a8e9 100644 --- a/designs/codegen/cli.md +++ b/designs/codegen/cli.md @@ -102,6 +102,13 @@ Smithy process's `PATH`. Smithy passes no arguments other than those in When invoked by Smithy, the CLI reads one JSON AST document from standard input. The document represents the model after projection transforms have been applied. +Smithy serializes only what a shape introduces, so shapes that use mixins arrive +without their inherited members, traits, and properties, and traits added to +inherited members arrive as `apply` statements. The CLI resolves mixins while +loading the model, following the rules of the +[Smithy mixins specification](https://smithy.io/2.0/spec/mixins.html), so builds +do not need the `flattenAndRemoveMixins` transform. + The presence of `SMITHY_PLUGIN_DIR` identifies an invocation by the `run` plugin. Generated files are written beneath this directory, which Smithy also uses as the process's working directory. The CLI MUST NOT write generated files outside it, From 85d815192006d3e4ec674c23ea02584aa8b67dff Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Sun, 13 Sep 2026 18:28:20 -0400 Subject: [PATCH 09/10] fix(codegen): Exit 1 when the model file cannot be read A missing --model path was pre-checked and reported as an invocation error with exit 2, while a file that existed but could not be read raised OSError and exited 1. The design classifies I/O failures as 1, and tools that distinguish usage errors from runtime failures, including the Smithy CLI, treat a missing input file as the latter. Drop the pre-check so every unreadable model exits 1 with a message that names the path and the cause. Also state in the design that model failures return 1, which the implementation already did but the text left implicit. --- designs/codegen/cli.md | 6 ++++-- packages/smithy-python/src/smithy_python/cli.py | 11 ++++++++--- packages/smithy-python/tests/unit/test_cli.py | 13 +++++++------ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/designs/codegen/cli.md b/designs/codegen/cli.md index d84f5a8e9..a4322de82 100644 --- a/designs/codegen/cli.md +++ b/designs/codegen/cli.md @@ -61,8 +61,10 @@ added when there is a need for it. The command MUST return zero after successful generation and non-zero when arguments, settings, the model, or generation are invalid. Diagnostics are -written to standard error. Invalid command syntax and invocation inputs return -2, while I/O and generation failures return 1. +written to standard error. Invalid command syntax and invocation inputs, such as +options that cannot be combined or a service that cannot be selected, return 2. +Model, I/O, and generation failures, including a model file that cannot be read, +return 1. ## Smithy `run` Plugin diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index 79cf26a62..05a35b2c8 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -132,9 +132,14 @@ def _resolve_invocation( output_dir = output_path if model_path is not None: - if not model_path.is_file(): - raise InvalidInvocationError(f"Model path is not a file: {model_path}") - model_source = model_path.read_bytes() + # A model that cannot be read is an I/O failure like any other, whether + # the path is missing, a directory, or unreadable. + try: + model_source = model_path.read_bytes() + except OSError as error: + raise OSError( + f"Cannot read model {model_path}: {error.strerror or error}" + ) from error else: model_stream = sys.stdin.buffer if stdin is None else stdin if environment.plugin_dir is None and model_stream.isatty(): diff --git a/packages/smithy-python/tests/unit/test_cli.py b/packages/smithy-python/tests/unit/test_cli.py index 74a2418cb..033a7bda8 100644 --- a/packages/smithy-python/tests/unit/test_cli.py +++ b/packages/smithy-python/tests/unit/test_cli.py @@ -163,7 +163,7 @@ def test_direct_invocation_rejects_interactive_model_input( ) -def test_invocation_reports_unreadable_model( +def test_invocation_reports_missing_model( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: missing = tmp_path / "missing.json" @@ -180,14 +180,15 @@ def test_invocation_reports_unreadable_model( ), environ={}, ) - == 2 + == 1 ) - assert f"Model path is not a file: {missing}" in capsys.readouterr().err + assert f"Cannot read model {missing}: No such file" in capsys.readouterr().err -def test_invocation_rejects_empty_model_path( +def test_invocation_reports_model_path_that_is_a_directory( tmp_path: Path, capsys: pytest.CaptureFixture[str] ) -> None: + # An empty path resolves to the current directory. assert ( main( ( @@ -200,9 +201,9 @@ def test_invocation_rejects_empty_model_path( ), environ={}, ) - == 2 + == 1 ) - assert "Model path is not a file: ." in capsys.readouterr().err + assert "Cannot read model .:" in capsys.readouterr().err def test_invocation_reports_model_io_error( From c7e2f6738697cf2b7a5c23cc42ebc367c1519222 Mon Sep 17 00:00:00 2001 From: jonathan343 Date: Tue, 15 Sep 2026 16:13:40 -0400 Subject: [PATCH 10/10] fix(codegen): Resolve the process environment once main substituted os.environ for a missing environ argument before passing it on, and PluginEnvironment.from_environ applied the same default again. Pass the argument through untouched so the fallback lives only in from_environ, and drop the now unused os import. --- packages/smithy-python/src/smithy_python/cli.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/smithy-python/src/smithy_python/cli.py b/packages/smithy-python/src/smithy_python/cli.py index 05a35b2c8..7a29baa25 100644 --- a/packages/smithy-python/src/smithy_python/cli.py +++ b/packages/smithy-python/src/smithy_python/cli.py @@ -5,7 +5,6 @@ from __future__ import annotations import argparse -import os import sys from collections.abc import Mapping, Sequence from dataclasses import dataclass @@ -44,11 +43,7 @@ def main( return error.code if isinstance(error.code, int) else 1 try: - _resolve_invocation( - args, - environ=os.environ if environ is None else environ, - stdin=stdin, - ) + _resolve_invocation(args, environ=environ, stdin=stdin) except InvalidInvocationError as error: sys.stderr.write(f"smithy-python: error: {error}\n") return 2 @@ -109,9 +104,10 @@ def _common_artifact_options() -> argparse.ArgumentParser: def _resolve_invocation( args: argparse.Namespace, *, - environ: Mapping[str, str], + environ: Mapping[str, str] | None, stdin: BinaryIO | None, ) -> _Invocation: + # PluginEnvironment falls back to os.environ, so the defaulting lives there. environment = PluginEnvironment.from_environ(environ) model_path: Path | None = args.model output_path: Path | None = args.output