Skip to content

Commit ee10946

Browse files
Merge pull request #1842 from gooddata/snapshot-master-c3c5284a-to-rel/dev
[bot] Merge master/c3c5284a into rel/dev
2 parents 5cc2aa8 + c3c5284 commit ee10946

11 files changed

Lines changed: 891 additions & 32 deletions

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py‎

Lines changed: 58 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,12 @@
3030
utc_now,
3131
)
3232
from gooddata_eval.core.chat.render import render_answer_text
33-
from gooddata_eval.core.chat.sse_client import ChatClient
33+
from gooddata_eval.core.chat.sse_client import ChatClient, ChatError
3434
from gooddata_eval.core.config import ReasoningEffort
3535
from gooddata_eval.core.models import (
3636
AgenticAssertionError,
3737
AgenticEvalOutcome,
38+
LoopExit,
3839
ReasoningStepEvent,
3940
ToolCallEvent,
4041
shift_and_index_events,
@@ -483,6 +484,12 @@ class AlertRunResult:
483484
response_id: str | None = None
484485
tool_call_events: list[ToolCallEvent] = field(default_factory=list)
485486
reasoning_step_events: list[ReasoningStepEvent] = field(default_factory=list)
487+
# Why the simulated-user loop stopped, and how many turns it took. Without these a run
488+
# that ran out of turns is indistinguishable from one that refused: both land on
489+
# alert_created=False, and every downstream check is `alert_created and ...`, so both
490+
# also report operator/threshold/metric/recipients as False.
491+
exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
492+
turns_used: int = 0
486493

487494

488495
@dataclass
@@ -681,8 +688,35 @@ def _run_once(conv_id: str) -> AlertRunResult:
681688
turns = 0
682689
steps = 0
683690

691+
# Defaults to BUDGET_EXHAUSTED: every other exit sets it explicitly, so a loop
692+
# that simply runs out of range() is correctly labelled without a trailing else.
693+
exit_reason = LoopExit.BUDGET_EXHAUSTED
694+
turns_used = 0
684695
for _iteration in range(max_iterations):
685-
chat_result = client.send_message(conv_id, current_question)
696+
turns_used = _iteration + 1
697+
try:
698+
chat_result = client.send_message(conv_id, current_question)
699+
except ChatError as exc:
700+
# Without this the exception escapes run_agentic_alert_skill entirely,
701+
# discarding every K-run already completed along with any exit_reason.
702+
print(f"[CHAT] send_message failed for conversation {conv_id}: {exc}")
703+
# The stream can break AFTER create_metric_alert already succeeded
704+
# server-side. That id only ever reached alert_id_to_delete from the
705+
# normal path below, so breaking here left the alert in the workspace for
706+
# the `finally` cleanup to miss -- a real object leaking out of a failed
707+
# run, not a reporting gap.
708+
partial = exc.partial_result
709+
if partial is not None:
710+
reasoning_steps.extend(partial.reasoning_steps or [])
711+
response_id = partial.response_id or response_id
712+
partial_alert_id, _, partial_tool_called = _extract_alert_call(partial.tool_call_events or [])
713+
if partial_tool_called:
714+
alert_id_to_delete = partial_alert_id
715+
exit_reason = LoopExit.CHAT_ERROR
716+
break
717+
# `turns_used` counts attempts (set above, so a failed send still shows the
718+
# loop reached this iteration); `turns`/`steps` count completed work, so they
719+
# advance only once send_message has actually returned.
686720
turns += 1
687721
steps += chat_result.reasoning_step_count
688722
reasoning_steps.extend(chat_result.reasoning_steps or [])
@@ -698,6 +732,7 @@ def _run_once(conv_id: str) -> AlertRunResult:
698732
alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
699733
if tool_called:
700734
alert_id_to_delete = alert_id
735+
exit_reason = LoopExit.SUCCESS
701736
break
702737
response_text = (chat_result.text_response or "").strip()
703738
if not response_text and chat_result.alert_proposals:
@@ -706,13 +741,24 @@ def _run_once(conv_id: str) -> AlertRunResult:
706741
response_text = render_answer_text(chat_result)
707742
# Stop if agent gave a completely empty response (stuck)
708743
if not response_text and not chat_result.tool_call_events:
744+
exit_reason = LoopExit.AGENT_SILENT
709745
break
710746
# Stop before generating a follow-up for the last iteration
711747
if _iteration >= max_iterations - 1:
712748
break
713-
follow_up = generate_simulated_alert_response(
714-
response_text, expected, conversation_history, question=question
715-
)
749+
# Recorded rather than raised, matching metric_skill and kda_skill. Letting it
750+
# propagate did keep a harness fault from being scored as a content failure,
751+
# but it also discarded the K-runs already completed -- and SIMULATED_USER_FAILED
752+
# achieves the same separation while keeping them, since reporting reads the
753+
# exit reason to classify the run as an error rather than an agent failure.
754+
try:
755+
follow_up = generate_simulated_alert_response(
756+
response_text, expected, conversation_history, question=question
757+
)
758+
except Exception as exc: # noqa: BLE001 -- harness-side fault; end only this run
759+
print(f"[SIM-USER] Simulated reply failed for conversation {conv_id}: {exc}")
760+
exit_reason = LoopExit.SIMULATED_USER_FAILED
761+
break
716762
# Record this exchange so the next call has full history
717763
conversation_history.append({"role": "assistant", "content": response_text})
718764
conversation_history.append({"role": "user", "content": follow_up})
@@ -740,6 +786,8 @@ def _run_once(conv_id: str) -> AlertRunResult:
740786
response_id=response_id,
741787
tool_call_events=all_tool_call_events,
742788
reasoning_step_events=all_reasoning_step_events,
789+
exit_reason=exit_reason,
790+
turns_used=turns_used,
743791
)
744792
finally:
745793
if alert_id_to_delete:
@@ -908,6 +956,11 @@ def _write_scores(ctx: RunTraceContext) -> None:
908956
"attributes_correct": ev.attributes_correct,
909957
"granularity_correct": ev.granularity_correct,
910958
"actual_alert_arguments": best.actual_alert_arguments,
959+
# Why the loop stopped. alert_created=False alone cannot tell a refusal from a run
960+
# that hit max_iterations while still on track -- see LoopExit.
961+
"exit_reason": best.exit_reason.value,
962+
"turns_used": best.turns_used,
963+
"max_iterations": max_iterations,
911964
**timeline_detail(best.tool_call_events, best.reasoning_step_events),
912965
}
913966

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py‎

Lines changed: 63 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
AgenticEvalOutcome,
6262
ChatResult,
6363
CreatedVisualization,
64+
LoopExit,
6465
ReasoningStepEvent,
6566
ToolCallEvent,
6667
shift_and_index_events,
@@ -168,6 +169,10 @@ class TurnResult(BaseModel):
168169
active_skills: list[str] = Field(default_factory=list)
169170
clarification_turns_used: int = 0
170171
output_correct: bool | None = None
172+
# Why this turn's clarification loop stopped -- see LoopExit. output_present=False alone
173+
# cannot separate a turn that ran out of clarification budget from one where the agent
174+
# went silent, and skill_success folds both into the same failure.
175+
exit_reason: LoopExit = LoopExit.BUDGET_EXHAUSTED
171176
mismatches: list[str] = Field(default_factory=list)
172177
clarifications: list[ClarificationRecord] = Field(default_factory=list)
173178
depends_on: list[str] = Field(default_factory=list)
@@ -219,6 +224,9 @@ def failure_reasons(self) -> list[str]:
219224
# What skill_routing was judged against -- without it, a turn showing
220225
# skill_routing=True and activated_skills=[] looks like a scoring bug.
221226
"active_skills",
227+
# Why the clarification loop ended on this turn, and how much of the budget it
228+
# took to get there -- exit_reason alone cannot be related to the limit without it.
229+
"exit_reason",
222230
"clarification_turns_used",
223231
"mismatches",
224232
"clarifications",
@@ -569,6 +577,10 @@ class ConversationResult:
569577
full_skill_coverage: bool
570578
conversation_success: bool
571579
total_clarification_turns: int
580+
# The configured per-turn clarification budget, so a reader can tell a turn that used
581+
# its whole allowance from one that stopped early. Every other agentic kind reports its
582+
# limit in detail; without this, conversation is the exception to that contract.
583+
max_clarification_turns: int = _DEFAULT_MAX_CLARIFICATION_TURNS
572584
total_steps: int = 0
573585
reasoning_steps: list[str] = field(default_factory=list)
574586
response_id: str | None = None
@@ -770,6 +782,10 @@ def run_agentic_conversation(
770782
# read as "nothing was active", which is a different claim.
771783
active_skills=sorted(active_skills),
772784
output_correct=False,
785+
# This turn's loop never ran at all -- a $ref pointing at an earlier
786+
# turn's output could not be resolved. Labelling it BUDGET_EXHAUSTED
787+
# (the field default) would claim it ran out of clarification turns.
788+
exit_reason=LoopExit.NOT_RUN,
773789
mismatches=[str(exc)],
774790
depends_on=list(turn.depends_on),
775791
)
@@ -790,6 +806,9 @@ def run_agentic_conversation(
790806
records: list[ClarificationRecord] = []
791807
answers_left = list(turn.set_answers)
792808

809+
# Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop
810+
# that simply runs out of range() is labelled correctly with no trailing else.
811+
turn_exit = LoopExit.BUDGET_EXHAUSTED
793812
for _iter in range(max_clarification_turns + 1):
794813
transcript.append(TranscriptEntry("user", current_message))
795814
incomplete = False
@@ -799,11 +818,36 @@ def run_agentic_conversation(
799818
chat_result = exc.partial_result or ChatResult()
800819
incomplete = True
801820
except ChatError as exc:
802-
# The stream can die after create_metric ran server-side; keep what the
803-
# accumulator saw so the cleanup still knows what to delete.
804-
if exc.partial_result is not None:
805-
created.record(exc.partial_result.tool_call_events or [])
806-
raise
821+
# Recorded rather than raised so the turns already completed keep their
822+
# results, and so this turn is distinguishable from one where the agent
823+
# simply failed to produce output. no_error below reads this back.
824+
print(f"[CHAT] send_message failed for conversation {conversation_id}: {exc}")
825+
partial = exc.partial_result
826+
if partial is not None:
827+
# Shifted and indexed exactly as a completed turn is, before anything
828+
# reads the events. What the stream managed to deliver before it died
829+
# is still this turn's work: left raw, its `call_ts` would restart near
830+
# zero and its indexes at zero, so `timeline_detail` would report a
831+
# late turn as overlapping the first one. The steps count for the same
832+
# reason -- they were taken.
833+
total_steps += partial.reasoning_step_count
834+
turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
835+
partial,
836+
turn_offset=turn_offset,
837+
tool_index_offset=tool_index_offset,
838+
reasoning_index_offset=reasoning_index_offset,
839+
)
840+
# The stream can die after create_metric ran server-side; keep what
841+
# the accumulator saw so the cleanup still knows what to delete.
842+
created.record(partial.tool_call_events or [])
843+
final_result = partial
844+
all_tool_calls.extend(partial.tool_call_events or [])
845+
conversation_tool_call_events.extend(partial.tool_call_events or [])
846+
conversation_reasoning_step_events.extend(partial.reasoning_step_events or [])
847+
reasoning_steps.extend(partial.reasoning_steps or [])
848+
response_id = partial.response_id or response_id
849+
turn_exit = LoopExit.CHAT_ERROR
850+
break
807851
final_result = chat_result
808852
total_steps += chat_result.reasoning_step_count
809853
turn_offset, tool_index_offset, reasoning_index_offset = shift_and_index_events(
@@ -833,6 +877,13 @@ def run_agentic_conversation(
833877
if _check_output_present(resolved_turn, chat_result):
834878
if incomplete:
835879
records.append(ClarificationRecord(kind="turn_incomplete", agent_message=clip(response_text)))
880+
turn_exit = LoopExit.SUCCESS
881+
break
882+
883+
# `incomplete` is excluded deliberately: a turn the server cut short is a
884+
# known stall with its own nudge path below, not an agent that said nothing.
885+
if not incomplete and not response_text and not chat_result.tool_call_events:
886+
turn_exit = LoopExit.AGENT_SILENT
836887
break
837888

838889
kind: ReplyRecordKind = "turn_incomplete" if incomplete else classify_reply(chat_result, response_text)
@@ -912,11 +963,15 @@ def run_agentic_conversation(
912963
expected_skill=turn.expected_skill,
913964
skill_routing=skill_routing,
914965
output_present=output_present,
915-
no_error=True, # SDK raises on errors; reaching here means no critical error.
966+
# A chat fault used to escape the whole run, so reaching here did mean no
967+
# error. Now that it is caught and recorded, this has to read it back --
968+
# otherwise a turn whose chat call failed reports no_error=True.
969+
no_error=turn_exit is not LoopExit.CHAT_ERROR,
916970
activated_skills=declared or [],
917971
active_skills=sorted(active_skills),
918972
clarification_turns_used=clarification_turns,
919973
output_correct=output_correct,
974+
exit_reason=turn_exit,
920975
mismatches=mismatches,
921976
clarifications=records,
922977
depends_on=list(turn.depends_on),
@@ -943,6 +998,7 @@ def run_agentic_conversation(
943998
full_skill_coverage=full_skill_coverage,
944999
conversation_success=conversation_success,
9451000
total_clarification_turns=total_clarification_turns,
1001+
max_clarification_turns=max_clarification_turns,
9461002
total_steps=total_steps,
9471003
reasoning_steps=reasoning_steps,
9481004
response_id=response_id,
@@ -969,6 +1025,7 @@ def _conversation_detail(result: ConversationResult) -> dict:
9691025
"lost_context_clarifications": result.lost_context_clarifications,
9701026
"stalled_turns": result.stalled_turns,
9711027
"total_clarification_turns": result.total_clarification_turns,
1028+
"max_clarification_turns": result.max_clarification_turns,
9721029
"judge_model": result.judge_model,
9731030
"turns": [tr.detail() for tr in result.turn_results],
9741031
**timeline_detail(result.tool_call_events, result.reasoning_step_events),
@@ -1033,7 +1090,6 @@ def evaluate_agentic_conversation(
10331090
failed_turns = {tr.turn_id: tr.failure_reasons() for tr in result.turn_results if not tr.context_success}
10341091

10351092
def _write_scores(ctx: RunTraceContext) -> None:
1036-
10371093
pt = ctx.trace(result.conversation_id)
10381094
with ctx.observe(
10391095
pt,

0 commit comments

Comments
 (0)