Merge pull request #247 from scaleapi/setup-agent-files-with-build-context

Adding command to compress files for Cloud Build

Author: RoxyFarhad

pyproject.toml

@@ -179,7 +179,8 @@ xfail_strict = true
 asyncio_mode = "auto"
 asyncio_default_fixture_loop_scope = "session"
 filterwarnings = [
-  "error"
+  "error",
+  "ignore::pydantic.warnings.PydanticDeprecatedSince20",
 ]
 
 [tool.pyright]

src/agentex/lib/cli/commands/agents.py

@@ -26,6 +26,8 @@
 from agentex.lib.cli.handlers.agent_handlers import (
     run_agent,
     build_agent,
+    parse_build_args,
+    prepare_cloud_build_context,
 )
 from agentex.lib.cli.handlers.deploy_handlers import (
     HelmError,
@@ -164,6 +166,101 @@ def build(
         raise typer.Exit(1) from e
 
 
+@agents.command(name="package")
+def package(
+    manifest: str = typer.Option(..., help="Path to the manifest you want to use"),
+    tag: str | None = typer.Option(
+        None,
+        "--tag",
+        "-t",
+        help="Image tag (defaults to deployment.image.tag from manifest, or 'latest')",
+    ),
+    output: str | None = typer.Option(
+        None,
+        "--output",
+        "-o",
+        help="Output filename for the tarball (defaults to <agent-name>-<tag>.tar.gz)",
+    ),
+    build_arg: builtins.list[str] | None = typer.Option(  # noqa: B008
+        None,
+        "--build-arg",
+        "-b",
+        help="Build argument in KEY=VALUE format (can be repeated)",
+    ),
+):
+    """
+    Package an agent's build context into a tarball for cloud builds.
+
+    Reads manifest.yaml, prepares build context according to include_paths and
+    dockerignore, then saves a compressed tarball to the current directory.
+
+    The tag defaults to the value in deployment.image.tag from the manifest.
+
+    Example:
+        agentex agents package --manifest manifest.yaml
+        agentex agents package --manifest manifest.yaml --tag v1.0
+    """
+    typer.echo(f"Packaging build context from manifest: {manifest}")
+
+    # Validate manifest exists
+    manifest_path = Path(manifest)
+    if not manifest_path.exists():
+        typer.echo(f"Error: manifest not found at {manifest_path}", err=True)
+        raise typer.Exit(1)
+
+    try:
+        # Prepare the build context (tag defaults from manifest if not provided)
+        build_context = prepare_cloud_build_context(
+            manifest_path=str(manifest_path),
+            tag=tag,
+            build_args=build_arg,
+        )
+
+        # Determine output filename using the resolved tag
+        if output:
+            output_filename = output
+        else:
+            output_filename = f"{build_context.agent_name}-{build_context.tag}.tar.gz"
+
+        # Save tarball to current working directory
+        output_path = Path.cwd() / output_filename
+        output_path.write_bytes(build_context.archive_bytes)
+
+        typer.echo(f"\nTarball saved to: {output_path}")
+        typer.echo(f"Size: {build_context.build_context_size_kb:.1f} KB")
+
+        # Output the build parameters needed for cloud build
+        typer.echo("\n" + "=" * 60)
+        typer.echo("Build Parameters for Cloud Build API:")
+        typer.echo("=" * 60)
+        typer.echo(f"  agent_name:      {build_context.agent_name}")
+        typer.echo(f"  image_name:      {build_context.image_name}")
+        typer.echo(f"  tag:             {build_context.tag}")
+        typer.echo(f"  context_file:    {output_path}")
+
+        if build_arg:
+            parsed_args = parse_build_args(build_arg)
+            typer.echo(f"  build_args:      {parsed_args}")
+
+        typer.echo("")
+        typer.echo("Command:")
+        build_args_str = ""
+        if build_arg:
+            build_args_str = " ".join(f'--build-arg "{arg}"' for arg in build_arg)
+            build_args_str = f" {build_args_str}"
+        typer.echo(
+            f'  sgp agentex build --context "{output_path}" '
+            f'--image-name "{build_context.image_name}" '
+            f'--tag "{build_context.tag}"{build_args_str}'
+        )
+        typer.echo("=" * 60)
+
+    except Exception as e:
+        typer.echo(f"Error packaging build context: {str(e)}", err=True)
+        logger.exception("Error packaging build context")
+        raise typer.Exit(1) from e
+
+
 @agents.command()
 def run(
     manifest: str = typer.Option(..., help="Path to the manifest you want to use"),

src/agentex/lib/cli/handlers/agent_handlers.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from typing import NamedTuple
 from pathlib import Path
 
 from rich.console import Console
@@ -8,7 +9,7 @@
 from agentex.lib.cli.debug import DebugConfig
 from agentex.lib.utils.logging import make_logger
 from agentex.lib.cli.handlers.run_handlers import RunError, run_agent as _run_agent
-from agentex.lib.sdk.config.agent_manifest import AgentManifest
+from agentex.lib.sdk.config.agent_manifest import AgentManifest, BuildContextManager
 
 logger = make_logger(__name__)
 console = Console()
@@ -18,6 +19,17 @@ class DockerBuildError(Exception):
     """An error occurred during docker build"""
 
 
+class CloudBuildContext(NamedTuple):
+    """Contains the prepared build context for cloud builds."""
+
+    archive_bytes: bytes
+    dockerfile_path: str
+    agent_name: str
+    tag: str
+    image_name: str
+    build_context_size_kb: float
+
+
 def build_agent(
     manifest_path: str,
     registry_url: str,
@@ -42,9 +54,7 @@ def build_agent(
         The image URL
     """
     agent_manifest = AgentManifest.from_yaml(file_path=manifest_path)
-    build_context_root = (
-        Path(manifest_path).parent / agent_manifest.build.context.root
-    ).resolve()
+    build_context_root = (Path(manifest_path).parent / agent_manifest.build.context.root).resolve()
 
     repository_name = repository_name or agent_manifest.agent.name
 
@@ -85,9 +95,7 @@ def build_agent(
                         key, value = arg.split("=", 1)
                         docker_build_args[key] = value
                     else:
-                        logger.warning(
-                            f"Invalid build arg format: {arg}. Expected KEY=VALUE"
-                        )
+                        logger.warning(f"Invalid build arg format: {arg}. Expected KEY=VALUE")
 
                 if docker_build_args:
                     docker_build_kwargs["build_args"] = docker_build_args
@@ -100,9 +108,7 @@ def build_agent(
             if push:
                 # Build and push in one step for multi-platform builds
                 logger.info("Building and pushing image...")
-                docker_build_kwargs["push"] = (
-                    True  # Push directly after build for multi-platform
-                )
+                docker_build_kwargs["push"] = True  # Push directly after build for multi-platform
                 docker.buildx.build(**docker_build_kwargs)
 
                 logger.info(f"Successfully built and pushed {image_name}")
@@ -146,15 +152,127 @@ def signal_handler(signum, _frame):
         shutting_down = True
         logger.info(f"Received signal {signum}, shutting down...")
         raise KeyboardInterrupt()
-    
+
     # Set up signal handling for the main thread
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
-    
+
     try:
         asyncio.run(_run_agent(manifest_path, debug_config))
     except KeyboardInterrupt:
         logger.info("Shutdown completed.")
         sys.exit(0)
     except RunError as e:
         raise RuntimeError(str(e)) from e
+
+
+def parse_build_args(build_args: list[str] | None) -> dict[str, str]:
+    """Parse build arguments from KEY=VALUE format to a dictionary.
+
+    Args:
+        build_args: List of build arguments in KEY=VALUE format
+
+    Returns:
+        Dictionary mapping keys to values
+    """
+    result: dict[str, str] = {}
+    if not build_args:
+        return result
+
+    for arg in build_args:
+        if "=" in arg:
+            key, value = arg.split("=", 1)
+            result[key] = value
+        else:
+            logger.warning(f"Invalid build arg format: {arg}. Expected KEY=VALUE")
+
+    return result
+
+
+def prepare_cloud_build_context(
+    manifest_path: str,
+    tag: str | None = None,
+    build_args: list[str] | None = None,
+) -> CloudBuildContext:
+    """Prepare the build context for cloud-based container builds.
+
+    Reads the manifest, prepares the build context by copying files according to
+    the include_paths and dockerignore, then creates a compressed tar.gz archive
+    ready for upload to a cloud build service.
+
+    Args:
+        manifest_path: Path to the agent manifest file
+        tag: Image tag override (if None, reads from manifest's deployment.image.tag)
+        build_args: List of build arguments in KEY=VALUE format
+
+    Returns:
+        CloudBuildContext containing the archive bytes, dockerfile path, and metadata
+    """
+    agent_manifest = AgentManifest.from_yaml(file_path=manifest_path)
+    build_context_root = (Path(manifest_path).parent / agent_manifest.build.context.root).resolve()
+
+    agent_name = agent_manifest.agent.name
+    dockerfile_path = agent_manifest.build.context.dockerfile
+
+    # Validate that the Dockerfile exists
+    full_dockerfile_path = build_context_root / dockerfile_path
+    if not full_dockerfile_path.exists():
+        raise FileNotFoundError(
+            f"Dockerfile not found at: {full_dockerfile_path}\n"
+            f"Check that 'build.context.dockerfile' in your manifest points to an existing file."
+        )
+    if not full_dockerfile_path.is_file():
+        raise ValueError(
+            f"Dockerfile path is not a file: {full_dockerfile_path}\n"
+            f"'build.context.dockerfile' must point to a file, not a directory."
+        )
+
+    # Get tag and repository from manifest if not provided
+    if tag is None:
+        if agent_manifest.deployment and agent_manifest.deployment.image:
+            tag = agent_manifest.deployment.image.tag
+        else:
+            tag = "latest"
+
+    # Get repository name from manifest (just the repo name, not the full registry URL)
+    if agent_manifest.deployment and agent_manifest.deployment.image:
+        repository = agent_manifest.deployment.image.repository
+        if repository:
+            # Extract just the repo name (last part after any slashes)
+            image_name = repository.split("/")[-1]
+        else:
+            image_name = "<repository>"
+    else:
+        image_name = "<repository>"
+
+    logger.info(f"Agent: {agent_name}")
+    logger.info(f"Image name: {image_name}")
+    logger.info(f"Build context root: {build_context_root}")
+    logger.info(f"Dockerfile: {dockerfile_path}")
+    logger.info(f"Tag: {tag}")
+
+    if agent_manifest.build.context.include_paths:
+        logger.info(f"Include paths: {agent_manifest.build.context.include_paths}")
+
+    parsed_build_args = parse_build_args(build_args)
+    if parsed_build_args:
+        logger.info(f"Build args: {list(parsed_build_args.keys())}")
+
+    logger.info("Preparing build context...")
+
+    with agent_manifest.context_manager(build_context_root) as build_context:
+        # Compress the prepared context using the static zipped method
+        with BuildContextManager.zipped(root_path=build_context.path) as archive_buffer:
+            archive_bytes = archive_buffer.read()
+
+        build_context_size_kb = len(archive_bytes) / 1024
+        logger.info(f"Build context size: {build_context_size_kb:.1f} KB")
+
+        return CloudBuildContext(
+            archive_bytes=archive_bytes,
+            dockerfile_path=build_context.dockerfile_path,
+            agent_name=agent_name,
+            tag=tag,
+            image_name=image_name,
+            build_context_size_kb=build_context_size_kb,
+        )