6161 AgenticEvalOutcome ,
6262 ChatResult ,
6363 CreatedVisualization ,
64+ LoopExit ,
6465 ReasoningStepEvent ,
6566 ToolCallEvent ,
6667 shift_and_index_events ,
@@ -168,6 +169,10 @@ class TurnResult(BaseModel):
168169 active_skills : list [str ] = Field (default_factory = list )
169170 clarification_turns_used : int = 0
170171 output_correct : bool | None = None
172+ # Why this turn's clarification loop stopped -- see LoopExit. output_present=False alone
173+ # cannot separate a turn that ran out of clarification budget from one where the agent
174+ # went silent, and skill_success folds both into the same failure.
175+ exit_reason : LoopExit = LoopExit .BUDGET_EXHAUSTED
171176 mismatches : list [str ] = Field (default_factory = list )
172177 clarifications : list [ClarificationRecord ] = Field (default_factory = list )
173178 depends_on : list [str ] = Field (default_factory = list )
@@ -219,6 +224,9 @@ def failure_reasons(self) -> list[str]:
219224 # What skill_routing was judged against -- without it, a turn showing
220225 # skill_routing=True and activated_skills=[] looks like a scoring bug.
221226 "active_skills" ,
227+ # Why the clarification loop ended on this turn, and how much of the budget it
228+ # took to get there -- exit_reason alone cannot be related to the limit without it.
229+ "exit_reason" ,
222230 "clarification_turns_used" ,
223231 "mismatches" ,
224232 "clarifications" ,
@@ -569,6 +577,10 @@ class ConversationResult:
569577 full_skill_coverage : bool
570578 conversation_success : bool
571579 total_clarification_turns : int
580+ # The configured per-turn clarification budget, so a reader can tell a turn that used
581+ # its whole allowance from one that stopped early. Every other agentic kind reports its
582+ # limit in detail; without this, conversation is the exception to that contract.
583+ max_clarification_turns : int = _DEFAULT_MAX_CLARIFICATION_TURNS
572584 total_steps : int = 0
573585 reasoning_steps : list [str ] = field (default_factory = list )
574586 response_id : str | None = None
@@ -770,6 +782,10 @@ def run_agentic_conversation(
770782 # read as "nothing was active", which is a different claim.
771783 active_skills = sorted (active_skills ),
772784 output_correct = False ,
785+ # This turn's loop never ran at all -- a $ref pointing at an earlier
786+ # turn's output could not be resolved. Labelling it BUDGET_EXHAUSTED
787+ # (the field default) would claim it ran out of clarification turns.
788+ exit_reason = LoopExit .NOT_RUN ,
773789 mismatches = [str (exc )],
774790 depends_on = list (turn .depends_on ),
775791 )
@@ -790,6 +806,9 @@ def run_agentic_conversation(
790806 records : list [ClarificationRecord ] = []
791807 answers_left = list (turn .set_answers )
792808
809+ # Defaults to BUDGET_EXHAUSTED: every other exit assigns explicitly, so a loop
810+ # that simply runs out of range() is labelled correctly with no trailing else.
811+ turn_exit = LoopExit .BUDGET_EXHAUSTED
793812 for _iter in range (max_clarification_turns + 1 ):
794813 transcript .append (TranscriptEntry ("user" , current_message ))
795814 incomplete = False
@@ -799,11 +818,36 @@ def run_agentic_conversation(
799818 chat_result = exc .partial_result or ChatResult ()
800819 incomplete = True
801820 except ChatError as exc :
802- # The stream can die after create_metric ran server-side; keep what the
803- # accumulator saw so the cleanup still knows what to delete.
804- if exc .partial_result is not None :
805- created .record (exc .partial_result .tool_call_events or [])
806- raise
821+ # Recorded rather than raised so the turns already completed keep their
822+ # results, and so this turn is distinguishable from one where the agent
823+ # simply failed to produce output. no_error below reads this back.
824+ print (f"[CHAT] send_message failed for conversation { conversation_id } : { exc } " )
825+ partial = exc .partial_result
826+ if partial is not None :
827+ # Shifted and indexed exactly as a completed turn is, before anything
828+ # reads the events. What the stream managed to deliver before it died
829+ # is still this turn's work: left raw, its `call_ts` would restart near
830+ # zero and its indexes at zero, so `timeline_detail` would report a
831+ # late turn as overlapping the first one. The steps count for the same
832+ # reason -- they were taken.
833+ total_steps += partial .reasoning_step_count
834+ turn_offset , tool_index_offset , reasoning_index_offset = shift_and_index_events (
835+ partial ,
836+ turn_offset = turn_offset ,
837+ tool_index_offset = tool_index_offset ,
838+ reasoning_index_offset = reasoning_index_offset ,
839+ )
840+ # The stream can die after create_metric ran server-side; keep what
841+ # the accumulator saw so the cleanup still knows what to delete.
842+ created .record (partial .tool_call_events or [])
843+ final_result = partial
844+ all_tool_calls .extend (partial .tool_call_events or [])
845+ conversation_tool_call_events .extend (partial .tool_call_events or [])
846+ conversation_reasoning_step_events .extend (partial .reasoning_step_events or [])
847+ reasoning_steps .extend (partial .reasoning_steps or [])
848+ response_id = partial .response_id or response_id
849+ turn_exit = LoopExit .CHAT_ERROR
850+ break
807851 final_result = chat_result
808852 total_steps += chat_result .reasoning_step_count
809853 turn_offset , tool_index_offset , reasoning_index_offset = shift_and_index_events (
@@ -833,6 +877,13 @@ def run_agentic_conversation(
833877 if _check_output_present (resolved_turn , chat_result ):
834878 if incomplete :
835879 records .append (ClarificationRecord (kind = "turn_incomplete" , agent_message = clip (response_text )))
880+ turn_exit = LoopExit .SUCCESS
881+ break
882+
883+ # `incomplete` is excluded deliberately: a turn the server cut short is a
884+ # known stall with its own nudge path below, not an agent that said nothing.
885+ if not incomplete and not response_text and not chat_result .tool_call_events :
886+ turn_exit = LoopExit .AGENT_SILENT
836887 break
837888
838889 kind : ReplyRecordKind = "turn_incomplete" if incomplete else classify_reply (chat_result , response_text )
@@ -912,11 +963,15 @@ def run_agentic_conversation(
912963 expected_skill = turn .expected_skill ,
913964 skill_routing = skill_routing ,
914965 output_present = output_present ,
915- no_error = True , # SDK raises on errors; reaching here means no critical error.
966+ # A chat fault used to escape the whole run, so reaching here did mean no
967+ # error. Now that it is caught and recorded, this has to read it back --
968+ # otherwise a turn whose chat call failed reports no_error=True.
969+ no_error = turn_exit is not LoopExit .CHAT_ERROR ,
916970 activated_skills = declared or [],
917971 active_skills = sorted (active_skills ),
918972 clarification_turns_used = clarification_turns ,
919973 output_correct = output_correct ,
974+ exit_reason = turn_exit ,
920975 mismatches = mismatches ,
921976 clarifications = records ,
922977 depends_on = list (turn .depends_on ),
@@ -943,6 +998,7 @@ def run_agentic_conversation(
943998 full_skill_coverage = full_skill_coverage ,
944999 conversation_success = conversation_success ,
9451000 total_clarification_turns = total_clarification_turns ,
1001+ max_clarification_turns = max_clarification_turns ,
9461002 total_steps = total_steps ,
9471003 reasoning_steps = reasoning_steps ,
9481004 response_id = response_id ,
@@ -969,6 +1025,7 @@ def _conversation_detail(result: ConversationResult) -> dict:
9691025 "lost_context_clarifications" : result .lost_context_clarifications ,
9701026 "stalled_turns" : result .stalled_turns ,
9711027 "total_clarification_turns" : result .total_clarification_turns ,
1028+ "max_clarification_turns" : result .max_clarification_turns ,
9721029 "judge_model" : result .judge_model ,
9731030 "turns" : [tr .detail () for tr in result .turn_results ],
9741031 ** timeline_detail (result .tool_call_events , result .reasoning_step_events ),
@@ -1033,7 +1090,6 @@ def evaluate_agentic_conversation(
10331090 failed_turns = {tr .turn_id : tr .failure_reasons () for tr in result .turn_results if not tr .context_success }
10341091
10351092 def _write_scores (ctx : RunTraceContext ) -> None :
1036-
10371093 pt = ctx .trace (result .conversation_id )
10381094 with ctx .observe (
10391095 pt ,
0 commit comments