feat(config): send [model.headers] on every call to the model endpoint, agents' included

.agentenv/config.example.toml

@@ -21,6 +21,13 @@ default_a2a_agent_id = "a2a-default"   # what a deploy_agent step that names no
 # base_url = "http://localhost:4000"   # a model proxy on this machine; local containers reach it as host.docker.internal
 #                                      # on Linux, bind it to the Docker bridge address and use that: http://172.17.0.1:4000
 # api_key  = "env:MY_PROXY_KEY"
+#
+# Hugging Face Inference Providers instead: a fine-grained token with only "Make calls to Inference
+# Providers", since this key reaches every agent. The headers go with every call to this endpoint.
+# base_url = "https://router.huggingface.co/v1"
+# api_key  = "env:HF_INFERENCE_TOKEN"
+# [model.headers]
+# "X-HF-Bill-To" = "my-org"   # bill an organization instead of the token's account
 
 [runner]
 impl = "agent_env.runner.local_runner:LocalRunner"

AGENTS.md

@@ -69,6 +69,9 @@ or one of them only where it is used.
   this machine that local containers call, a model proxy say, has to listen there: bind it to the
   bridge address and use that address in its URL. Binding `0.0.0.0` also works, but exposes the
   service to the network.
+- `[model.headers]` go with every call to the configured endpoint and nowhere else: `extra_headers` on
+  agent-env's own calls, `LITELLM_EXTRA_HEADERS` (JSON, written by core, never read) for agents and the
+  unit-test verifier. On `sail_vm` with `inject_model_key`, Sail sets them at egress like the key.
 - A host-mode `install_agent` (no `container_name`) on a `local` sandbox runs the agent's
   `install_commands_host` on this machine, with your privileges. Every command a local sandbox runs
   carries `AGENTENV_SANDBOX=<sandbox id>`, and teardown stops the processes that carry it, then removes

packages/agentenv-protocol/README.md

@@ -679,8 +679,9 @@ multi-operation custom extensions, and advanced ASGI-lifespan plus
 common SDK-operation override hooks.
 
 The agent examples make real OpenAI-compatible model calls. Set
-`LITELLM_API_KEY` and, when needed, `LITELLM_BASE_URL`; their typed agent config
-selects the model and system prompt for each deployment. Tests inject a fake
+`LITELLM_API_KEY` and, when needed, `LITELLM_BASE_URL`; they also send the headers in
+`LITELLM_EXTRA_HEADERS`, a JSON object AgentEnv sets when the endpoint needs any. Their typed
+agent config selects the model and system prompt for each deployment. Tests inject a fake
 model client, so the example suite remains offline and deterministic.
 
 ### Object transfer

packages/agentenv-protocol/examples/reference_model.py

@@ -42,13 +42,18 @@ def _request(self, *, api_key: str | None = None) -> tuple[str, dict[str, str]]:
             raise RuntimeError(
                 "LITELLM_BASE_URL must be set before running the agent"
             )
+        # The endpoint's headers are for the endpoint AgentEnv configured, never one this client was pointed at.
+        configured = os.getenv("LITELLM_BASE_URL") or ""
+        headers = (json.loads(os.getenv("LITELLM_EXTRA_HEADERS") or "{}")
+                   if base_url.rstrip("/") == configured.rstrip("/") else {})
         base_url = base_url.rstrip("/")
         endpoint = (
             f"{base_url}/chat/completions"
             if base_url.endswith("/v1")
             else f"{base_url}/v1/chat/completions"
         )
-        return endpoint, {"Authorization": f"Bearer {api_key}"}
+        own = {k: v for k, v in headers.items() if k.lower() != "authorization"}
+        return endpoint, {**own, "Authorization": f"Bearer {api_key}"}
 
     async def complete(
         self,

packages/agentenv-protocol/tests/test_a2a_agent_examples.py

@@ -64,9 +64,24 @@ def _load_example(name: str) -> ModuleType:
 
 
 @pytest.mark.asyncio
-async def test_reference_model_forwards_generic_metadata(
+@pytest.mark.parametrize("endpoint_headers, configured_url, sent_headers", [
+    (None, "https://models.example/v1", {"Authorization": "Bearer request-scoped-key"}),
+    ('{"X-Bill-To": "acme", "authorization": "Bearer theirs"}', "https://models.example/v1",
+     {"X-Bill-To": "acme", "Authorization": "Bearer request-scoped-key"}),
+    ('{"X-Bill-To": "acme"}', "https://models.example/v1/", {"X-Bill-To": "acme", "Authorization": "Bearer request-scoped-key"}),
+    ('{"X-Bill-To": "acme"}', "https://configured.example/v1", {"Authorization": "Bearer request-scoped-key"}),
+], ids=["no-endpoint-headers", "endpoint-headers", "trailing-slash", "another-endpoint"])
+async def test_reference_model_forwards_generic_metadata_and_the_endpoints_headers(
     monkeypatch: pytest.MonkeyPatch,
+    endpoint_headers: str | None,
+    configured_url: str,
+    sent_headers: dict[str, str],
 ) -> None:
+    monkeypatch.setenv("LITELLM_BASE_URL", configured_url)
+    if endpoint_headers is None:
+        monkeypatch.delenv("LITELLM_EXTRA_HEADERS", raising=False)
+    else:
+        monkeypatch.setenv("LITELLM_EXTRA_HEADERS", endpoint_headers)
     captured: dict[str, Any] = {}
     class Response:
         def raise_for_status(self) -> None:
@@ -107,7 +122,7 @@ async def post(self, url: str, **kwargs: Any) -> Response:
         "client": {"timeout": 120},
         "request": {
             "url": "https://models.example/v1/chat/completions",
-            "headers": {"Authorization": "Bearer request-scoped-key"},
+            "headers": sent_headers,
             "json": {
                 "model": "test-model",
                 "messages": [{"role": "user", "content": "hello"}],

src/agent_env/a2a_agent/a2a_agent.py

@@ -13,6 +13,7 @@
 from agentenv_protocol.a2a_agent import STANDARD_EXTENSIONS
 
 from agent_env.config import get_config
+from agent_env.config.model import MODEL_HEADERS_ENV
 from agent_env.entity_refs import EntityRef
 from agent_env.providers.sandbox_providers.sandbox import (
     SANDBOX_LABEL,
@@ -436,6 +437,11 @@ async def deploy(
             resolved_env["LITELLM_API_KEY"] = config.get_litellm_api_key()
         if "LITELLM_BASE_URL" not in resolved_env and "LITELLM_BASE_URL" not in self.default_env_vars:
             resolved_env["LITELLM_BASE_URL"] = config.get_litellm_base_url()
+        if MODEL_HEADERS_ENV not in resolved_env and MODEL_HEADERS_ENV not in self.default_env_vars:
+            headers = config.get_model_headers(
+                resolved_env.get("LITELLM_BASE_URL", self.default_env_vars.get("LITELLM_BASE_URL")))
+            if headers:
+                resolved_env[MODEL_HEADERS_ENV] = json.dumps(headers)
         if "LITELLM_BASE_URL" in resolved_env:
             resolved_env["LITELLM_BASE_URL"] = (provider or sandbox).url_from_sandbox(resolved_env["LITELLM_BASE_URL"])
 

src/agent_env/config/describe.py

@@ -59,8 +59,9 @@
 # debugging round-trip, under-masking prints a credential into a terminal and a paste. On
 # that trade `auth` is in even though it also catches `author` — a store `config` table is
 # where `authorization = "Bearer ..."` lives, and a masked author is the cheaper mistake.
+# `headers` masks a whole table of HTTP headers: a name like `X-Api-Key` doesn't say it holds a key.
 _SECRET_KEY_PARTS = ("password", "passwd", "passphrase", "secret", "token", "credential",
-                     "api_key", "private_key", "auth", "bearer")
+                     "api_key", "private_key", "auth", "bearer", "headers")
 _URI_USERINFO = re.compile(r"(?P<scheme>[a-zA-Z][\w+.-]*://)(?P<userinfo>[^/@\s]+)@")
 
 

src/agent_env/config/model.py

@@ -1,19 +1,29 @@
-"""Resolution of the ``[model]`` config.toml block: endpoint, default/per-role models, and
-provider params. A ``ModelConfig`` value object parsed from the ``[model]`` table; a
-``secret_resolver`` is injected per call so secret references resolve lazily."""
+"""Resolution of the ``[model]`` config.toml block: endpoint, default/per-role models,
+provider params, and the endpoint's headers. A ``ModelConfig`` value object parsed from the
+``[model]`` table; a ``secret_resolver`` is injected per call so secret references resolve lazily."""
 
 from __future__ import annotations
 
+import logging
 import os
+import re
 from dataclasses import dataclass, field, fields
 from enum import StrEnum
 from typing import Optional
+from urllib.parse import urlparse
 
 from agent_env.config.errors import ConfigError
-from agent_env.config.loader import SecretResolver, interpolate
+from agent_env.config.loader import ENV_REF_PREFIX, SECRET_REF_PREFIX, SecretResolver, interpolate
+
+logger = logging.getLogger(__name__)
 
 _ENV_BASE_URL = "LITELLM_BASE_URL"
 _ENV_API_KEY = "LITELLM_API_KEY"
+# Agents get ``[model.headers]`` here as a JSON object; unlike its two siblings, agent-env never reads it.
+MODEL_HEADERS_ENV = "LITELLM_EXTRA_HEADERS"
+# RFC 9110: a header name is a token; a value is visible ASCII, spaces and tabs (no CR or LF to split it on).
+_HEADER_NAME = re.compile(r"[!#$%&'*+.^_`|~0-9A-Za-z-]+\Z")
+_HEADER_VALUE = re.compile(r"[\t\x20-\x7e]*\Z")
 
 
 class ModelParam(StrEnum):
@@ -57,13 +67,14 @@ class ModelConfig:
     default: Optional[str] = None
     roles: dict = field(default_factory=dict)
     params: dict = field(default_factory=dict)
+    headers: dict = field(default_factory=dict)  # values may hold unresolved env:/secret: references
 
     @classmethod
     def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> ModelConfig:
         """Build from the raw ``[model]`` section: reject unknown keys and reserved
         ``[model.params]`` keys, then resolve ``env:``/``secret:`` refs — except
-        ``api_key``, which keeps its raw reference and resolves lazily per call
-        (an eager secret fetch would make every ``[model]`` read need credentials)."""
+        ``api_key`` and ``[model.headers]``, which keep their raw references and resolve
+        lazily per call (an eager secret fetch would make every ``[model]`` read need credentials)."""
         allowed = {f.name for f in fields(cls)}
         unknown = set(section) - allowed
         if unknown:
@@ -74,8 +85,13 @@ def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> Mode
                 f"[model.params] may not set reserved keys {sorted(reserved)} "
                 "(agent-env sets these on the model call)"
             )
+        headers = section.get("headers", {})
+        if not isinstance(headers, dict) or not all(isinstance(v, str) for v in headers.values()):
+            raise ConfigError("[model.headers] maps header names to strings (an env:/secret: reference is fine)")
+        for name, value in headers.items():
+            _check_header(name, None if value.startswith((ENV_REF_PREFIX, SECRET_REF_PREFIX)) else value)
         cfg = interpolate(
-            {k: v for k, v in section.items() if k != "api_key"},
+            {k: v for k, v in section.items() if k not in ("api_key", "headers")},
             secret_resolver=secret_resolver,
         )
         return cls(
@@ -84,6 +100,7 @@ def from_section(cls, section: dict, *, secret_resolver: SecretResolver) -> Mode
             default=cfg.get("default"),
             roles=cfg.get("roles", {}),
             params=cfg.get("params", {}),
+            headers=dict(headers),
         )
 
     def model_for_role(self, role: str) -> Optional[str]:
@@ -106,6 +123,21 @@ def _resolved_api_key(self, secret_resolver: SecretResolver) -> Optional[str]:
             return None
         return interpolate(self.api_key, secret_resolver=secret_resolver)
 
+    def headers_for(self, base_url: Optional[str], *, secret_resolver: SecretResolver) -> dict[str, str]:
+        """``[model.headers]``, resolved, for a call or an agent pointed at ``base_url``: none unless
+        that is the configured endpoint, since a header may carry a credential."""
+        if not self.headers or not base_url:
+            return {}
+        configured_base = os.getenv(_ENV_BASE_URL) or self.base_url
+        if not configured_base or base_url.rstrip("/") != configured_base.rstrip("/"):
+            logger.info("Not sending [model.headers] to %s, which is not the configured model endpoint",
+                        urlparse(base_url).hostname or base_url)
+            return {}
+        headers = interpolate(dict(self.headers), secret_resolver=secret_resolver)
+        for name, value in headers.items():
+            _check_header(name, value)
+        return headers
+
     def resolve_call(
         self,
         model: str,
@@ -149,6 +181,21 @@ def resolve_call(
         # The configured key is scoped to the configured endpoint (and native routing):
         # a base_override pointing elsewhere must not inherit it, or a caller-supplied
         # URL would receive the configured proxy's credential.
-        if key is None and (api_base is None or api_base == configured_base):
+        if key is None and (api_base is None or api_base.rstrip("/") == (configured_base or "").rstrip("/")):
             key = self._resolved_api_key(secret_resolver)
-        return ModelCallConfig(api_key=key, api_base=api_base, params=dict(self.params))
+        params = dict(self.params)
+        headers = self.headers_for(api_base, secret_resolver=secret_resolver)
+        if headers:
+            explicit = params.get("extra_headers", {})
+            overridden = {name.lower() for name in explicit}
+            params["extra_headers"] = {**{k: v for k, v in headers.items() if k.lower() not in overridden}, **explicit}
+        return ModelCallConfig(api_key=key, api_base=api_base, params=params)
+
+
+def _check_header(name: str, value: Optional[str]) -> None:
+    """Refuse a ``[model.headers]`` entry HTTP can't carry (``value`` None checks the name alone): a non-ASCII
+    value would fail inside an agent's client, and a CR or LF would split one header into two."""
+    if not _HEADER_NAME.match(name):
+        raise ConfigError(f"[model.headers] {name!r} is not an HTTP header name")
+    if value is not None and not _HEADER_VALUE.match(value):
+        raise ConfigError(f"[model.headers] {name!r} has a value HTTP can't carry: only visible ASCII, spaces and tabs")

src/agent_env/config/runtime.py

@@ -407,6 +407,9 @@ def get_model_params(self, overrides: dict | None = None) -> dict:
             params.update(config_loader.interpolate(overrides, secret_resolver=self._resolve_secret))
         return params
 
+    def get_model_headers(self, base_url: str | None) -> dict[str, str]:
+        return self._model_config().headers_for(base_url, secret_resolver=self._resolve_secret)
+
     def resolve_model_call(
         self,
         model: str,

src/agent_env/providers/sandbox_providers/sail_vm/README.md

@@ -189,16 +189,20 @@ asyncio.run(main())
 - **Reconnect:** `get_sandbox` restores the ports and the applied egress policy. A policy it can't
   represent leaves `network_policy` unknown, and image loading then fails closed.
 
-## The agent's model key
+## The agent's model key and headers
 
-With `inject_model_key` on, an agent's `LITELLM_API_KEY` never enters its Sailbox:
+With `inject_model_key` on, an agent's `LITELLM_API_KEY` never enters its Sailbox, and neither do the
+values of the endpoint's `[model.headers]`:
 
 1. The key is stored as a Sail secret named `AGENTENV_LITELLM_<sha256(key)[:32]>`, one per distinct key.
+   Each header value is a secret of its own, `AGENTENV_LITELLM_H_<sha256(name, value)[:32]>`.
 2. The Sailbox is created with a saved egress policy. On requests to `LITELLM_BASE_URL`'s host, the policy
-   sets the `authorization: Bearer …` and `x-api-key` headers from that secret. Sail adds the key as each
-   request leaves the box.
+   sets the `authorization: Bearer …` and `x-api-key` headers from the key's secret. On those under its path
+   (`/v1/` for `https://host/v1`), it also sets each header from its own, so another service on the same host
+   never gets them. Sail adds them as each request leaves the box.
 3. Inside the box, every command and file agent-env sends carries the placeholder
-   `sail-injected-model-key` in place of the key.
+   `sail-injected-model-key` in place of the key. `LITELLM_EXTRA_HEADERS` keeps the header names with
+   `sail-injected-header` as every value, which the policy overwrites.
 4. A `docker` shim on the box gives every container the VM's CA bundle (`SSL_CERT_FILE`,
    `REQUESTS_CA_BUNDLE`, `NODE_EXTRA_CA_CERTS`, `CURL_CA_BUNDLE`). Sail terminates TLS for the model host
    with its own CA, so containers need it to trust those requests.
@@ -207,12 +211,13 @@ Requirements and behaviour:
 
 - **Endpoint:** the model endpoint must be HTTPS and reachable from Sail.
 - **Key length:** keys shorter than 16 characters are refused, because scrubbing replaces the key
-  wherever it appears.
-- **Other keys are refused:** a Sailbox refuses any command or file that would carry a model key Sail
-  doesn't inject for it. One example is an agent deployed into a sandbox it didn't create.
+  wherever it appears. Headers are replaced as a whole, so their values have no minimum.
+- **Headers the key sets:** a `[model.headers]` entry named `authorization` or `x-api-key` is refused.
+- **Other keys are refused:** a Sailbox refuses any command or file that would carry a model key, or
+  header values, Sail doesn't inject for it. One example is an agent deployed into a sandbox it didn't create.
 - **Turning it off:** set `inject_model_key = false` to pass keys in as other providers do.
-- **Cleanup:** terminate deletes the Sailbox's policy. The secret is kept, so a concurrent launch with the
-  same key never loses it. To remove secrets for retired keys, sweep `AGENTENV_LITELLM_*` with
+- **Cleanup:** terminate deletes the Sailbox's policy. The secrets are kept, so a concurrent launch with the
+  same key or headers never loses them. To remove secrets for retired keys, sweep `AGENTENV_LITELLM_*` with
   `sail secret list`.
 
 ## Attribution and cost