Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions src/agent_env/a2a_agent/validator.py
Original file line number Diff line number Diff line change
Expand Up @@ -727,7 +727,7 @@ def _upload_install_test_image_fixture(agent: "A2AAgent"):
ts = int(time.time())
universe_id = derive_id(agent.id, f"validate-install-image-v{agent.version}-{ts}")
config = get_config()
s3_url = config.get_object_store_for(universe_id).object_url(f"{config.get_artifact_key_prefix()}a2a_validator/install_test_image/{ts}/")
prefix_url = config.get_object_store_for(universe_id).object_url(f"{config.get_artifact_key_prefix()}a2a_validator/install_test_image/{ts}/")

with tempfile.NamedTemporaryFile("w", suffix=".Dockerfile", delete=False) as f:
f.write("FROM ubuntu:24.04\n")
Expand All @@ -736,9 +736,9 @@ def _upload_install_test_image_fixture(agent: "A2AAgent"):
universe = FileArtifactUniverse.put_bundled(
id=universe_id,
files={"Dockerfile": Path(tmp_path)},
s3_url=s3_url,
prefix_url=prefix_url,
)
logger.info(f"Uploaded install-test image universe '{universe.id}' v{universe.version} to {s3_url}")
logger.info(f"Uploaded install-test image universe '{universe.id}' v{universe.version} to {prefix_url}")
return universe
finally:
try:
Expand Down
7 changes: 7 additions & 0 deletions src/agent_env/artifact/artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,13 @@
from agent_env.artifact.store import ArtifactQuery


def _write_twin(data: dict[str, Any], legacy: str, neutral: str, value: Any) -> None:
"""Write a renamed field under both its legacy and its neutral key, when the dump has it under either: readers
of either spelling keep working, and a field the dump excluded stays excluded."""
if legacy in data or neutral in data:
data[legacy] = data[neutral] = value


class Artifact(BaseModel):
"""Base class for all environment artifacts. Immutable - each modification creates a new version."""

Expand Down
24 changes: 17 additions & 7 deletions src/agent_env/artifact/artifacts/cli.py
Original file line number Diff line number Diff line change
@@ -1,19 +1,19 @@
"""CliArtifact — packages a CLI tool (entrypoint + supporting files) as a versioned artifact.

A CliArtifact references one FileArtifactUniverse bundling every file in the CLI
directory tree, colocated under a single S3 prefix (`artifacts/cli/<key_segment(id)>/<version>/`).
directory tree, colocated under a single object-store prefix (`artifacts/cli/<key_segment(id)>/<version>/`).
The `entrypoint` field is a relative path within that prefix that consumers
(e.g. an agent gateway) chmod +x and symlink onto PATH.
"""

from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING, Literal, Optional
from typing import TYPE_CHECKING, Any, Literal, Optional

from pydantic import Field
from pydantic import ConfigDict, Field, model_serializer

from agent_env.artifact.artifact import Artifact
from agent_env.artifact.artifact import Artifact, _write_twin
from agent_env.config import get_config
from agent_env.store.ids import derive_id, key_segment

Expand All @@ -22,6 +22,8 @@


class CliArtifact(Artifact):
model_config = ConfigDict(populate_by_name=True)

type: Literal["cli"] = "cli"

cli_files_id: str = Field(description="ID of the FileArtifactUniverse bundling every file in the CLI tree")
Expand All @@ -32,6 +34,14 @@ class CliArtifact(Artifact):
env_id: Optional[str] = Field(default=None, description="Source env id this CLI was generated from, if any")
env_version: Optional[int] = Field(default=None, description="Source env version this CLI was generated from, if any")

# No return annotation: pydantic builds the serialization schema from one, and a dict drops the fields.
@model_serializer(mode="wrap")
def _serialize(self, handler: Any):
# Dual-write, from the attribute: `alias=` emits one spelling, which one depending on the caller's `by_alias`.
data = handler(self)
_write_twin(data, "cli_s3_url", "cli_object_url", self.cli_object_url)
return data

@classmethod
def put(
cls,
Expand Down Expand Up @@ -62,15 +72,15 @@ def put(
store = get_artifact_store()
version = store.next_version(id)
key = f"{get_config().get_artifact_key_prefix()}artifacts/cli/{key_segment(id)}/{version}/"
cli_s3_url = get_config().get_object_store_for(id).object_url(key)
cli_object_url = get_config().get_object_store_for(id).object_url(key)

universe = FileArtifactUniverse.put_bundled(id=derive_id(id, "files"), files=files, s3_url=cli_s3_url)
universe = FileArtifactUniverse.put_bundled(id=derive_id(id, "files"), files=files, prefix_url=cli_object_url)

instance = cls(
id=id,
version=version,
cli_files_id=universe.id,
cli_s3_url=cli_s3_url,
cli_object_url=cli_object_url,
entrypoint=entrypoint,
command_name=command_name,
env_id=env_id,
Expand Down
49 changes: 35 additions & 14 deletions src/agent_env/artifact/artifacts/docker_image.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
"""Docker image artifact for storing Docker images in S3."""
"""Docker image artifact for storing Docker images in the object store."""

from __future__ import annotations

Expand All @@ -16,14 +16,15 @@
from dataclasses import dataclass
from importlib.metadata import version as pkg_version
from pathlib import Path
from typing import Callable, ClassVar, Literal
from typing import Any, Callable, ClassVar, Literal
from urllib.parse import urlparse

from pydantic import Field
from pydantic import ConfigDict, Field, model_serializer

from agent_env.artifact.artifact import Artifact
from agent_env.artifact.artifact import Artifact, _write_twin
from agent_env.store.ids import fs_safe, image_repository, is_local_id
from agent_env.store.image_store.oci_registry_credentials import is_loopback_host, registry_host_from_ref
from agent_env.utils.deprecation import OMITTED, renamed_keyword

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -62,7 +63,9 @@ def _git_clone_commands(owner: str, repo: str, ref: str | None, token: str | Non
]

class DockerImageArtifact(Artifact):
"""A Docker image artifact stored as tar.gz in S3."""
"""A Docker image artifact stored as tar.gz in the object store."""

model_config = ConfigDict(populate_by_name=True)

DOCKER_SAVE_TIMEOUT_SECONDS: ClassVar[int] = 900

Expand All @@ -72,6 +75,15 @@ class DockerImageArtifact(Artifact):
tar_gz_object_url: str = Field(alias="tar_gz_s3_url", description="Object-store locator of the tar.gz file")
build_context_object_url: str | None = Field(default=None, alias="build_context_s3_url", description="Object-store locator of the build context tar.gz")

# No return annotation: pydantic builds the serialization schema from one, and a dict drops the fields.
@model_serializer(mode="wrap")
def _serialize(self, handler: Any):
# Dual-write, from the attributes: `alias=` emits one spelling, which one depending on the caller's `by_alias`.
data = handler(self)
_write_twin(data, "tar_gz_s3_url", "tar_gz_object_url", self.tar_gz_object_url)
_write_twin(data, "build_context_s3_url", "build_context_object_url", self.build_context_object_url)
return data

@classmethod
def put(
cls,
Expand Down Expand Up @@ -150,8 +162,8 @@ def put(
id,
description=description,
image_name=image_ref,
tar_gz_s3_url=tar_gz_object_url,
build_context_s3_url=build_context_object_url,
tar_gz_object_url=tar_gz_object_url,
build_context_object_url=build_context_object_url,
)

@classmethod
Expand All @@ -161,13 +173,22 @@ def put_tar(
*,
description: str,
image_name: str,
tar_gz_s3_url: str,
build_context_s3_url: str | None = None,
tar_gz_object_url: str | None = None,
build_context_object_url: str | None = None,
tar_gz_s3_url: str | None = OMITTED,
build_context_s3_url: str | None = OMITTED,
) -> "DockerImageArtifact":
from agent_env.artifact.store import get_artifact_store
from agent_env.config import get_config

for url in (tar_gz_s3_url, build_context_s3_url):
owner = "DockerImageArtifact.put_tar"
tar_gz_object_url = renamed_keyword(owner, "tar_gz_object_url", tar_gz_object_url, "tar_gz_s3_url", tar_gz_s3_url)
build_context_object_url = renamed_keyword(
owner, "build_context_object_url", build_context_object_url, "build_context_s3_url", build_context_s3_url
)
if tar_gz_object_url is None:
raise TypeError(f"{owner}() missing required keyword argument: 'tar_gz_object_url'")
for url in (tar_gz_object_url, build_context_object_url):
if url:
get_config().check_object_url(id, url)
store = get_artifact_store()
Expand All @@ -177,8 +198,8 @@ def put_tar(
version=version,
description=description,
image_name=image_name,
tar_gz_s3_url=tar_gz_s3_url,
build_context_s3_url=build_context_s3_url,
tar_gz_object_url=tar_gz_object_url,
build_context_object_url=build_context_object_url,
)
return store.put_document(instance)

Expand Down Expand Up @@ -311,8 +332,8 @@ async def put_from_github(
id=id,
description=f"Built from GitHub: {dockerfile_github_url}",
image_name=image_ref,
tar_gz_s3_url=tar_gz_object_url,
build_context_s3_url=build_context_object_url,
tar_gz_object_url=tar_gz_object_url,
build_context_object_url=build_context_object_url,
)
logger.info(f"put_from_github: created artifact id={artifact.id} version={artifact.version}")

Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/artifact/artifacts/environment.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,9 @@ class EnvironmentArtifact(Artifact):
description="Legacy unpinned wrapped-FileArtifact id",
)

# No return annotation: pydantic builds the serialization schema from one, and a dict drops the fields.
@model_serializer(mode="wrap")
def _serialize(self, handler: Any) -> dict[str, Any]:
def _serialize(self, handler: Any):
# `@model_serializer(mode="wrap")` is the only hook invoked by both
# `model_dump` and the Rust-implemented `model_dump_json`.
data = handler(self)
Expand Down
3 changes: 2 additions & 1 deletion src/agent_env/artifact/artifacts/environment_universe.py
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,9 @@ class EnvironmentUniverseArtifact(Universe):
description="Legacy unpinned metadata FileArtifact ids",
)

# No return annotation: pydantic builds the serialization schema from one, and a dict drops the fields.
@model_serializer(mode="wrap")
def _serialize(self, handler: Any) -> dict[str, Any]:
def _serialize(self, handler: Any):
# `@model_serializer(mode="wrap")` is the only hook invoked by both
# `model_dump` and the Rust-implemented `model_dump_json`.
data = handler(self)
Expand Down
35 changes: 22 additions & 13 deletions src/agent_env/artifact/artifacts/file.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,22 @@
"""Generic file artifact for storing arbitrary files in S3."""
"""Generic file artifact for storing arbitrary files in the object store."""

from __future__ import annotations

import mimetypes
import os
from typing import TYPE_CHECKING, ClassVar, Literal, Self
from typing import TYPE_CHECKING, Any, ClassVar, Literal, Self

from pydantic import Field
from pydantic import ConfigDict, Field, model_serializer

from agent_env.artifact.artifact import Artifact
from agent_env.artifact.artifact import Artifact, _write_twin
from agent_env.config import get_config

if TYPE_CHECKING:
from agent_env.bundle.authoring import AuthoringContext


class FileArtifact(Artifact):
"""A generic file artifact that stores any file in S3.
"""A generic file artifact that stores any file in the object store.

Clients define their own schemas and upload files (JSON, images, etc.).
When loaded, returns raw bytes for the client to parse as needed.
Expand All @@ -34,13 +34,23 @@ class FileArtifact(Artifact):
emails = json.loads(data) # Client parses as needed
"""

model_config = ConfigDict(populate_by_name=True)

toml_keys: ClassVar[dict[str, type]] = {"description": str} # what an artifact.toml may set
type: Literal["file"] = "file"
description: str = Field(description="Human-readable description of the artifact contents")
filename: str = Field(description="Original filename (preserved for reference)")
content_type: str = Field(description="MIME type of the file")
object_url: str = Field(alias="s3_url", description="Object-store locator where the file is stored")

# No return annotation: pydantic builds the serialization schema from one, and a dict drops the fields.
@model_serializer(mode="wrap")
def _serialize(self, handler: Any):
# Dual-write, from the attribute: `alias=` emits one spelling, which one depending on the caller's `by_alias`.
data = handler(self)
_write_twin(data, "s3_url", "object_url", self.object_url)
return data

@classmethod
def put(
cls,
Expand All @@ -63,7 +73,7 @@ def put(
# Use put_object_file (boto3 managed multipart upload) instead of
# put_object (s3.put_object — hard 5 GB single-object limit). Streams
# directly from disk so large files don't get loaded into RAM either.
s3_url = store.put_object_file(
stored_url = store.put_object_file(
artifact_type="file",
id=id,
version=version,
Expand All @@ -79,7 +89,7 @@ def put(
description=description,
filename=filename,
content_type=content_type,
s3_url=s3_url,
object_url=stored_url,
)
return store.put_document(instance)

Expand All @@ -99,7 +109,7 @@ def put_bytes(
store = get_artifact_store()
version = store.next_version(id)

s3_url = store.put_object(
stored_url = store.put_object(
artifact_type="file",
id=id,
version=version,
Expand All @@ -114,7 +124,7 @@ def put_bytes(
description=description,
filename=filename,
content_type=content_type,
s3_url=s3_url,
object_url=stored_url,
)
return store.put_document(instance)

Expand Down Expand Up @@ -174,7 +184,7 @@ def put_at(
description=description,
filename=filename,
content_type=content_type,
s3_url=stored_url,
object_url=stored_url,
)
return artifact_store.put_document(instance)

Expand All @@ -196,8 +206,7 @@ def put_existing(
from agent_env.artifact.store import get_artifact_store

store = get_config().get_object_store_to_write(object_url, id)
location = object_url.partition("://")[2] or object_url
if "/" not in location or location.endswith("/"):
if object_url.endswith("/"):
raise ValueError(f"object_url must point at an object, not a prefix: {object_url!r}")
metadata = store.get_object_metadata_at(object_url)
if metadata is None:
Expand All @@ -215,6 +224,6 @@ def put_existing(
description=description,
filename=filename,
content_type=content_type,
s3_url=object_url,
object_url=object_url,
)
return artifact_store.put_document(instance)
Loading
Loading