diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index d6cc72c1..c2111ace 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -19,7 +19,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from ..records import UnsupportedStepTypeError, record_from_step -from .config import ScorerInvokeConfig +from .config import ScorerInvokeConfig, scorer_invoke_runtime_enabled logger = logging.getLogger(__name__) @@ -367,6 +367,8 @@ class GalileoLunaClient: Environment Variables: GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth. GALILEO_LUNA_INVOKE_URL: Luna scorer invoke URL or service root (required). + GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME: Set to ``enabled`` to send + structured runtime data. GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify Luna invoke TLS. AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback. GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS: HTTP pooled connection expiry. @@ -414,6 +416,7 @@ def __init__( ) self.api_secret = resolved_api_secret + self.runtime_enabled = scorer_invoke_runtime_enabled() self.luna_invoke_url = _normalize_luna_invoke_url(resolved_luna_invoke_url) self.luna_invoke_ca_file = ( luna_invoke_ca_file @@ -559,7 +562,7 @@ async def invoke( selected_data=selected_data, payload_field=selected_data_payload_field, ) - if step is not None and scorer_version_id is not None + if self.runtime_enabled and step is not None and scorer_version_id is not None else None ) request_body = ScorerInvokeRequest( @@ -573,7 +576,7 @@ async def invoke( tools=step.tools if step is not None else None, ), record=record, - execution_context=execution_context, + execution_context=execution_context if self.runtime_enabled else None, config=invoke_config, ).to_dict() diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py index 52c315ec..e1b13782 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/config.py @@ -2,6 +2,7 @@ from __future__ import annotations +import os from typing import Literal from agent_control_evaluators import EvaluatorConfig @@ -12,6 +13,12 @@ LunaPayloadField = Literal["input", "output"] _NUMERIC_OPERATORS = frozenset({"gt", "gte", "lt", "lte"}) +SCORER_INVOKE_RUNTIME_FLAG_ENV = "GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME" + + +def scorer_invoke_runtime_enabled() -> bool: + """Return whether structured runtime data is enabled for Luna requests.""" + return os.getenv(SCORER_INVOKE_RUNTIME_FLAG_ENV) == "enabled" class ScorerInvokeConfig(BaseModel): diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 3d986216..7589270e 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -17,6 +17,11 @@ LUNA_ENV = { "GALILEO_API_SECRET_KEY": "test-secret", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", + "GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME": "enabled", +} +LEGACY_LUNA_ENV = { + "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", } SCORER_ID = "3d45ef0d-5f14-4f1a-a8f1-8ab758da18b4" SCORER_VERSION_ID = "07fb9c96-9752-4cf5-a253-1a396100e9d2" @@ -885,6 +890,60 @@ def handler(request: httpx.Request) -> httpx.Response: "config": {"request_timeout_seconds": 8.0}, } + @pytest.mark.asyncio + @pytest.mark.parametrize("flag_value", [None, "disabled", "Enabled"]) + async def test_client_uses_legacy_request_without_runtime_flag( + self, flag_value: str | None + ) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + from agent_control_evaluator_galileo.luna.client import GalileoExecutionContext + + captured: dict[str, object] = {} + + def handler(request: httpx.Request) -> httpx.Response: + captured["body"] = json.loads(request.content.decode()) + return httpx.Response(200, json={"score": 0.8, "status": "success"}) + + step = Step( + type="llm", + name="answer", + input="step input", + output="step output", + ground_truth={"expected": "answer"}, + tools=[{"name": "search", "input_schema": {}}], + ) + env = LEGACY_LUNA_ENV.copy() + if flag_value is not None: + env["GALILEO_FEATURE_FLAG_SCORER_INVOKE_RUNTIME"] = flag_value + with patch.dict(os.environ, env, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + try: + await client.invoke( + scorer_id=SCORER_ID, + scorer_version_id=SCORER_VERSION_ID, + input="selected input", + output="selected output", + step=step, + execution_context=GalileoExecutionContext(organization_id="org-1"), + ) + finally: + await client.close() + + body = captured["body"] + assert isinstance(body, dict) + assert body["scorer_id"] == SCORER_ID + assert body["scorer_version_id"] == SCORER_VERSION_ID + assert body["inputs"] == { + "query": "selected input", + "response": "selected output", + "ground_truth": {"expected": "answer"}, + "tools": [{"name": "search", "input_schema": {}}], + } + assert "record" not in body + assert "execution_context" not in body + @pytest.mark.asyncio async def test_client_omits_record_when_step_has_no_pinned_version(self) -> None: from agent_control_evaluator_galileo.luna import GalileoLunaClient