Merge pull request #1778 from gooddata/snapshot-master-58485168-to-rel/dev

[bot] Merge master/58485168 into rel/dev

Author: yenkins-admin

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -29,6 +29,7 @@
     ReasoningStepEvent,
     ToolCallEvent,
     build_latency_breakdown,
+    shift_and_index_events,
 )
 
 try:
@@ -526,21 +527,14 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 chat_result = client.send_message(conv_id, current_question)
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
-                for tc in chat_result.tool_call_events or []:
-                    if tc.call_ts is not None:
-                        tc.call_ts += turn_offset
-                    if tc.result_ts is not None:
-                        tc.result_ts += turn_offset
-                    if tc.index is not None:
-                        tc.index += tool_index_offset
-                for rs in chat_result.reasoning_step_events or []:
-                    rs.ts += turn_offset
-                    rs.index += reasoning_index_offset
+                turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+                    chat_result,
+                    turn_offset=turn_offset,
+                    tool_index_offset=tool_index_offset,
+                    reasoning_index_offset=reasoning_index_offset,
+                )
                 all_tool_call_events.extend(chat_result.tool_call_events or [])
                 all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
-                tool_index_offset += len(chat_result.tool_call_events or [])
-                reasoning_index_offset += len(chat_result.reasoning_step_events or [])
-                turn_offset += chat_result.turn_wall_clock_sec or 0.0
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -31,6 +31,7 @@
     ReasoningStepEvent,
     ToolCallEvent,
     build_latency_breakdown,
+    shift_and_index_events,
 )
 from gooddata_eval.core.scoring import (
     check_filters,
@@ -368,22 +369,15 @@ def run_agentic_conversation(
             for _iter in range(max_clarification_turns + 1):
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
-                for tc in chat_result.tool_call_events or []:
-                    if tc.call_ts is not None:
-                        tc.call_ts += turn_offset
-                    if tc.result_ts is not None:
-                        tc.result_ts += turn_offset
-                    if tc.index is not None:
-                        tc.index += tool_index_offset
-                for rs in chat_result.reasoning_step_events or []:
-                    rs.ts += turn_offset
-                    rs.index += reasoning_index_offset
+                turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+                    chat_result,
+                    turn_offset=turn_offset,
+                    tool_index_offset=tool_index_offset,
+                    reasoning_index_offset=reasoning_index_offset,
+                )
                 all_tool_calls.extend(chat_result.tool_call_events or [])
                 conversation_tool_call_events.extend(chat_result.tool_call_events or [])
                 conversation_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
-                tool_index_offset += len(chat_result.tool_call_events or [])
-                reasoning_index_offset += len(chat_result.reasoning_step_events or [])
-                turn_offset += chat_result.turn_wall_clock_sec or 0.0
                 reasoning_steps.extend(chat_result.reasoning_steps or [])
                 response_id = chat_result.response_id or response_id
 

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -18,7 +18,13 @@
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError, LLMJudge, score_run
-from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome
+from gooddata_eval.core.models import (
+    AgenticAssertionError,
+    AgenticEvalOutcome,
+    ReasoningStepEvent,
+    ToolCallEvent,
+    build_latency_breakdown,
+)
 from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
 
 _DEFAULT_K = 1
@@ -71,6 +77,8 @@ class GeneralQuestionResult:
     # excluded from pass@K and from Langfuse scoring rather than counted as a failure:
     # scoring it 0 would publish a verdict the judge never gave.
     judge_error: str | None = None
+    tool_call_events: list[ToolCallEvent] = field(default_factory=list)
+    reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
 
 
 @dataclass
@@ -132,6 +140,8 @@ def _run_single_general_question(
         # agent still answered, and that measurement is the one worth keeping.
         timings=PhaseTimings(agent_s=agent_elapsed, judge_s=judge_elapsed),
         judge_error=verdict.error,
+        tool_call_events=list(chat_result.tool_call_events or []),
+        reasoning_step_events=list(chat_result.reasoning_step_events or []),
     )
 
 
@@ -303,6 +313,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         "judge_passed": best.passed,
         "judge_reasoning": best.reasoning,
         "actual_output": best.actual_output,
+        "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
         # Only present when it happened, so the usual JSON shape is unchanged. A
         # pass@K computed over fewer runs than --runs asked for is a weaker result and
         # the report has to say so.

packages/gooddata-eval/src/gooddata_eval/core/agentic/kda_skill.py

@@ -18,7 +18,15 @@
 )
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ToolCallEvent
+from gooddata_eval.core.models import (
+    AgenticAssertionError,
+    AgenticEvalOutcome,
+    ChatResult,
+    ReasoningStepEvent,
+    ToolCallEvent,
+    build_latency_breakdown,
+    shift_and_index_events,
+)
 
 _log = logging.getLogger(__name__)
 
@@ -170,6 +178,8 @@ class KdaRunResult:
     turn_wall_clock_sec: float | None = None
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
+    tool_call_events: list[ToolCallEvent] = field(default_factory=list)
+    reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
 
 
 @dataclass
@@ -240,6 +250,22 @@ def _run_once(conv_id: str) -> KdaRunResult:
         current_question = question
         reasoning_steps: list[str] = []
         response_id: str | None = None
+        all_tool_call_events: list[ToolCallEvent] = []
+        all_reasoning_step_events: list[ReasoningStepEvent] = []
+        turn_offset = 0.0  # each turn's call_ts/ts restarts near 0 -- shift by prior turns' wall time
+        tool_index_offset = 0
+        reasoning_index_offset = 0
+
+        def _accumulate(result: ChatResult) -> None:
+            nonlocal turn_offset, tool_index_offset, reasoning_index_offset
+            turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+                result,
+                turn_offset=turn_offset,
+                tool_index_offset=tool_index_offset,
+                reasoning_index_offset=reasoning_index_offset,
+            )
+            all_tool_call_events.extend(result.tool_call_events or [])
+            all_reasoning_step_events.extend(result.reasoning_step_events or [])
 
         for iteration in range(max_iterations):
             try:
@@ -250,13 +276,15 @@ def _run_once(conv_id: str) -> KdaRunResult:
                 if partial is not None:
                     reasoning_steps.extend(partial.reasoning_steps or [])
                     response_id = partial.response_id or response_id
+                    _accumulate(partial)
                     create_args, execute_result = _extract_kda_calls(partial.tool_call_events or [])
                     if create_args is not None:
                         turn_wall_clock_sec = partial.turn_wall_clock_sec
                 turn_completed = False
                 break
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
+            _accumulate(chat_result)
             create_args, execute_result = _extract_kda_calls(chat_result.tool_call_events or [])
             response_text = (chat_result.text_response or "").strip()
             turn_completed = chat_result.stream_ended and bool(response_text)
@@ -295,6 +323,8 @@ def _run_once(conv_id: str) -> KdaRunResult:
             turn_wall_clock_sec=turn_wall_clock_sec,
             reasoning_steps=reasoning_steps,
             response_id=response_id,
+            tool_call_events=all_tool_call_events,
+            reasoning_step_events=all_reasoning_step_events,
         )
 
     try:
@@ -451,6 +481,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         "disambiguated": ev.disambiguated,
         "actual_create_args": best.actual_create_args,
         "actual_execute_result": best.actual_execute_result,
+        "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
     }
 
     if not summary.pass_at_k:

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -28,6 +28,7 @@
     ReasoningStepEvent,
     ToolCallEvent,
     build_latency_breakdown,
+    shift_and_index_events,
 )
 from gooddata_eval.core.timing import PhaseTimings, log_timer, sum_timings
 
@@ -321,21 +322,14 @@ def _execute_single_metric_run(
             timings.agent_s += agent_elapsed
             reasoning_steps.extend(chat_result.reasoning_steps or [])
             response_id = chat_result.response_id or response_id
-            for tc in chat_result.tool_call_events or []:
-                if tc.call_ts is not None:
-                    tc.call_ts += turn_offset
-                if tc.result_ts is not None:
-                    tc.result_ts += turn_offset
-                if tc.index is not None:
-                    tc.index += tool_index_offset
-            for rs in chat_result.reasoning_step_events or []:
-                rs.ts += turn_offset
-                rs.index += reasoning_index_offset
+            turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+                chat_result,
+                turn_offset=turn_offset,
+                tool_index_offset=tool_index_offset,
+                reasoning_index_offset=reasoning_index_offset,
+            )
             all_tool_call_events.extend(chat_result.tool_call_events or [])
             all_reasoning_step_events.extend(chat_result.reasoning_step_events or [])
-            tool_index_offset += len(chat_result.tool_call_events or [])
-            reasoning_index_offset += len(chat_result.reasoning_step_events or [])
-            turn_offset += chat_result.turn_wall_clock_sec or 0.0
             for metric_id in _extract_created_metric_ids(chat_result.tool_call_events or []):
                 if metric_id not in created_metric_ids:
                     created_metric_ids.append(metric_id)

packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py

@@ -16,7 +16,13 @@
 )
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import AgenticAssertionError, AgenticEvalOutcome, ToolCallEvent
+from gooddata_eval.core.models import (
+    AgenticAssertionError,
+    AgenticEvalOutcome,
+    ReasoningStepEvent,
+    ToolCallEvent,
+    build_latency_breakdown,
+)
 
 _DEFAULT_K = 1
 
@@ -59,6 +65,8 @@ class SearchResult:
     tool_call_names: list[str]
     reasoning_steps: list[str] = field(default_factory=list)
     response_id: str | None = None
+    tool_call_events: list[ToolCallEvent] = field(default_factory=list)
+    reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
 
 
 @dataclass
@@ -103,6 +111,8 @@ def run_agentic_search_tool(
                     tool_call_names=[tc.function_name for tc in tcs],
                     reasoning_steps=list(chat_result.reasoning_steps or []),
                     response_id=chat_result.response_id,
+                    tool_call_events=list(chat_result.tool_call_events or []),
+                    reasoning_step_events=list(chat_result.reasoning_step_events or []),
                 )
             )
         finally:
@@ -124,6 +134,8 @@ def run_agentic_search_tool(
                         tool_call_names=[tc.function_name for tc in tcs],
                         reasoning_steps=list(chat_result.reasoning_steps or []),
                         response_id=chat_result.response_id,
+                        tool_call_events=list(chat_result.tool_call_events or []),
+                        reasoning_step_events=list(chat_result.reasoning_step_events or []),
                     )
                 )
             finally:
@@ -233,6 +245,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
         "tool_selected": best.tool_selected,
         "tool_correct": best.tool_correct,
         "tool_call_names": best.tool_call_names,
+        "latency_breakdown": build_latency_breakdown(best.tool_call_events, best.reasoning_step_events),
     }
 
     if not summary.pass_at_k:

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -34,6 +34,7 @@
     ReasoningStepEvent,
     ToolCallEvent,
     build_latency_breakdown,
+    shift_and_index_events,
 )
 from gooddata_eval.core.scoring import get_dimension_uri_set, get_metric_uri_set, uri_to_display_name
 
@@ -192,23 +193,16 @@ def _execute_single_run(
     for iteration in range(max_iterations):
         total_turns += 1.0
         total_steps += float(current_result.reasoning_step_count)
-        for tc in current_result.tool_call_events:
-            if tc.call_ts is not None:
-                tc.call_ts += turn_offset
-            if tc.result_ts is not None:
-                tc.result_ts += turn_offset
-            if tc.index is not None:
-                tc.index += tool_index_offset
-        for rs in current_result.reasoning_step_events:
-            rs.ts += turn_offset
-            rs.index += reasoning_index_offset
+        turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
+            current_result,
+            turn_offset=turn_offset,
+            tool_index_offset=tool_index_offset,
+            reasoning_index_offset=reasoning_index_offset,
+        )
         all_tool_call_events.extend(current_result.tool_call_events)
         all_reasoning_step_events.extend(current_result.reasoning_step_events)
-        tool_index_offset += len(current_result.tool_call_events)
-        reasoning_index_offset += len(current_result.reasoning_step_events)
         reasoning_steps.extend(current_result.reasoning_steps or [])
         response_id = current_result.response_id or response_id
-        turn_offset += current_result.turn_wall_clock_sec or 0.0
 
         viz_produced = bool(current_result.created_visualizations and current_result.created_visualizations.objects)
         if viz_produced:

packages/gooddata-eval/src/gooddata_eval/core/evaluators/alert_skill.py

@@ -6,7 +6,7 @@
 
 from gooddata_eval.core.evaluators._deep_subset import deep_subset
 from gooddata_eval.core.evaluators.base import ItemEvaluation
-from gooddata_eval.core.models import ChatResult, DatasetItem
+from gooddata_eval.core.models import ChatResult, DatasetItem, build_latency_breakdown
 
 _TRIGGER_MAP = {"Every time": "ALWAYS", "One time": "ONCE"}
 
@@ -72,7 +72,13 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
             return ItemEvaluation(
                 passed=False,
                 rank_key=(False,) * 7,
-                detail={"alert_created": False, "automation_id": None},
+                detail={
+                    "alert_created": False,
+                    "automation_id": None,
+                    "latency_breakdown": build_latency_breakdown(
+                        chat_result.tool_call_events, chat_result.reasoning_step_events
+                    ),
+                },
             )
 
         args = tool_event.parsed_arguments()
@@ -145,5 +151,8 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
                 # lets a caller (e.g. a cleanup step) delete the exact object
                 # created instead of diffing the workspace catalog before/after.
                 "automation_id": automation_id,
+                "latency_breakdown": build_latency_breakdown(
+                    chat_result.tool_call_events, chat_result.reasoning_step_events
+                ),
             },
         )

packages/gooddata-eval/src/gooddata_eval/core/evaluators/metric_skill.py

@@ -2,7 +2,7 @@
 """Evaluator for metric_skill: agent must create the correct metric via create_metric tool call."""
 
 from gooddata_eval.core.evaluators.base import ItemEvaluation
-from gooddata_eval.core.models import ChatResult, DatasetItem
+from gooddata_eval.core.models import ChatResult, DatasetItem, build_latency_breakdown
 
 
 def _find_create_metric(chat_result: ChatResult):
@@ -28,7 +28,15 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
             return ItemEvaluation(
                 passed=False,
                 rank_key=(False, False, False),
-                detail={"metric_created": False, "maql_correct": False, "format_correct": False, "metric_id": None},
+                detail={
+                    "metric_created": False,
+                    "maql_correct": False,
+                    "format_correct": False,
+                    "metric_id": None,
+                    "latency_breakdown": build_latency_breakdown(
+                        chat_result.tool_call_events, chat_result.reasoning_step_events
+                    ),
+                },
             )
 
         result = tool_event.parsed_result()
@@ -59,5 +67,8 @@ def evaluate(self, item: DatasetItem, chat_result: ChatResult) -> ItemEvaluation
                 # delete the exact object created instead of diffing the workspace
                 # catalog before/after and guessing by name.
                 "metric_id": payload.get("metric_id"),
+                "latency_breakdown": build_latency_breakdown(
+                    chat_result.tool_call_events, chat_result.reasoning_step_events
+                ),
             },
         )