Skip to content

Commit b992cf9

Browse files
TomkessTomkess
authored
Merge pull request #1750 from gooddata/feat/agentic-reasoning-outcome-remaining-kinds
feat(gooddata-eval): return AgenticEvalOutcome from every agentic kind, not just 3 of 8
2 parents a0e06bc + b7cb7d5 commit b992cf9

20 files changed

Lines changed: 1050 additions & 79 deletions

‎packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -86,12 +86,12 @@ def _dispatch_agentic(
8686
model_version_override: str | None,
8787
reasoning_effort: ReasoningEffort | None = None,
8888
agent_id: str | None = None,
89-
) -> AgenticEvalOutcome | list[str] | None:
89+
) -> AgenticEvalOutcome:
9090
"""Call the appropriate evaluate_agentic_* function for the item's test_kind.
9191
92-
Returns whatever that function returns -- alert_skill/metric_skill/conversation return
93-
an AgenticEvalOutcome; the rest still return None
94-
(unchanged).
92+
Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
93+
conversation_id, response_id, detail) on success and attaches the same four attributes
94+
to its raised *AssertionError on failure -- no kind is exempt.
9595
"""
9696
kind = item.test_kind
9797
eo = item.expected_output
@@ -174,13 +174,14 @@ def _dispatch_agentic(
174174
**lf_kw,
175175
)
176176
elif kind == "agentic_kda_skill":
177-
evaluate_agentic_kda_skill(
177+
return evaluate_agentic_kda_skill(
178178
host=host,
179179
token=token,
180180
workspace_id=workspace_id,
181181
question=item.question,
182182
expected_output=eo if isinstance(eo, dict) else {},
183183
k=k,
184+
agent_id=agent_id,
184185
**lf_kw,
185186
)
186187
elif kind == "agentic_conversation":
@@ -240,19 +241,22 @@ def run_agentic_items(
240241
reasoning_steps = outcome.reasoning_steps
241242
conversation_id = outcome.conversation_id
242243
response_id = outcome.response_id
244+
detail = outcome.detail
243245
else:
244-
reasoning_steps, conversation_id, response_id = outcome, None, None
246+
reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {}
245247
item_report.pass_at_k = True
246248
item_report.runs = k
247249
item_report.reasoning_steps = reasoning_steps or []
248250
item_report.conversation_id = conversation_id
249251
item_report.response_id = response_id
252+
item_report.best_detail = detail or {}
250253
except AssertionError as exc:
251254
item_report.pass_at_k = False
252255
item_report.runs = k
253256
item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
254257
item_report.conversation_id = getattr(exc, "conversation_id", None)
255258
item_report.response_id = getattr(exc, "response_id", None)
259+
item_report.best_detail = getattr(exc, "detail", None) or {}
256260
print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
257261
except Exception as exc:
258262
item_report.error = f"{type(exc).__name__}: {exc}"

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -584,6 +584,7 @@ class AlertSkillAssertionError(AssertionError):
584584
reasoning_steps: list[str]
585585
conversation_id: str
586586
response_id: str | None
587+
detail: dict
587588

588589

589590
def evaluate_agentic_alert_skill(
@@ -697,9 +698,31 @@ def evaluate_agentic_alert_skill(
697698
exc.reasoning_steps = best.reasoning_steps
698699
exc.conversation_id = best.conversation_id
699700
exc.response_id = best.response_id
701+
exc.detail = {
702+
"alert_created": ev.alert_created,
703+
"operator_correct": ev.operator_correct,
704+
"threshold_correct": ev.threshold_correct,
705+
"trigger_correct": ev.trigger_correct,
706+
"filters_correct": ev.filters_correct,
707+
"metric_correct": ev.metric_correct,
708+
"recipients_correct": ev.recipients_correct,
709+
"actual_alert_arguments": best.actual_alert_arguments,
710+
}
700711
raise exc
712+
best = summary.best
713+
ev = best.eval
701714
return AgenticEvalOutcome(
702-
reasoning_steps=summary.best.reasoning_steps,
703-
conversation_id=summary.best.conversation_id,
704-
response_id=summary.best.response_id,
715+
reasoning_steps=best.reasoning_steps,
716+
conversation_id=best.conversation_id,
717+
response_id=best.response_id,
718+
detail={
719+
"alert_created": ev.alert_created,
720+
"operator_correct": ev.operator_correct,
721+
"threshold_correct": ev.threshold_correct,
722+
"trigger_correct": ev.trigger_correct,
723+
"filters_correct": ev.filters_correct,
724+
"metric_correct": ev.metric_correct,
725+
"recipients_correct": ev.recipients_correct,
726+
"actual_alert_arguments": best.actual_alert_arguments,
727+
},
705728
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -393,13 +393,32 @@ def run_agentic_conversation(
393393
)
394394

395395

396+
def _conversation_detail(result: ConversationResult) -> dict:
397+
return {
398+
"full_skill_coverage": result.full_skill_coverage,
399+
"total_clarification_turns": result.total_clarification_turns,
400+
"turns": [
401+
{
402+
"turn_id": tr.turn_id,
403+
"expected_skill": tr.expected_skill,
404+
"skill_routing": tr.skill_routing,
405+
"output_present": tr.output_present,
406+
"output_correct": tr.output_correct,
407+
"activated_skills": tr.activated_skills,
408+
}
409+
for tr in result.turn_results
410+
],
411+
}
412+
413+
396414
class ConversationAssertionError(AssertionError):
397415
"""Raised when a conversation evaluation fails."""
398416

399417
__tracebackhide__ = True
400418
reasoning_steps: list[str]
401419
conversation_id: str
402420
response_id: str | None
421+
detail: dict
403422

404423

405424
def evaluate_agentic_conversation(
@@ -511,9 +530,11 @@ def evaluate_agentic_conversation(
511530
exc.reasoning_steps = result.reasoning_steps
512531
exc.conversation_id = result.conversation_id
513532
exc.response_id = result.response_id
533+
exc.detail = _conversation_detail(result)
514534
raise exc
515535
return AgenticEvalOutcome(
516536
reasoning_steps=result.reasoning_steps,
517537
conversation_id=result.conversation_id,
518538
response_id=result.response_id,
539+
detail=_conversation_detail(result),
519540
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py‎

Lines changed: 40 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@
33

44
from __future__ import annotations
55

6-
from dataclasses import dataclass
6+
from dataclasses import dataclass, field
77

88
from gooddata_eval.core.chat.sse_client import ChatClient
99
from gooddata_eval.core.config import ReasoningEffort
1010
from gooddata_eval.core.evaluators._llm_judge import LLMJudge
11+
from gooddata_eval.core.models import AgenticEvalOutcome
1112

1213
_DEFAULT_K = 1
1314

@@ -52,6 +53,8 @@ class GeneralQuestionResult:
5253
passed: bool
5354
llm_judge_score: float
5455
reasoning: str
56+
reasoning_steps: list[str] = field(default_factory=list)
57+
response_id: str | None = None
5558

5659

5760
@dataclass
@@ -98,6 +101,8 @@ def run_agentic_general_question(
98101
passed=passed,
99102
llm_judge_score=llm_judge_score,
100103
reasoning=reasoning,
104+
reasoning_steps=list(chat_result.reasoning_steps or []),
105+
response_id=chat_result.response_id,
101106
)
102107
)
103108
finally:
@@ -120,6 +125,8 @@ def run_agentic_general_question(
120125
passed=passed,
121126
llm_judge_score=llm_judge_score,
122127
reasoning=reasoning,
128+
reasoning_steps=list(chat_result.reasoning_steps or []),
129+
response_id=chat_result.response_id,
123130
)
124131
)
125132
finally:
@@ -142,6 +149,10 @@ class GeneralQuestionAssertionError(AssertionError):
142149
"""Raised when a general-question evaluation fails."""
143150

144151
__tracebackhide__ = True
152+
reasoning_steps: list[str]
153+
conversation_id: str
154+
response_id: str | None
155+
detail: dict
145156

146157

147158
def evaluate_agentic_general_question(
@@ -160,8 +171,13 @@ def evaluate_agentic_general_question(
160171
model_version_override: str | None = None,
161172
run_metadata_extra: dict | None = None,
162173
reasoning_effort: ReasoningEffort | None = None,
163-
) -> None:
164-
"""Run general-question evaluation, log to Langfuse, and raise on failure."""
174+
) -> AgenticEvalOutcome:
175+
"""Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure.
176+
177+
Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
178+
AgenticEvalOutcome on success; on failure the same three values are attached to the
179+
raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``.
180+
"""
165181
from datetime import datetime as _dt # noqa: PLC0415
166182
from datetime import timezone as _tz # noqa: PLC0415
167183

@@ -223,6 +239,26 @@ def evaluate_agentic_general_question(
223239

224240
if not summary.pass_at_k:
225241
best = summary.best
226-
raise GeneralQuestionAssertionError(
242+
exc = GeneralQuestionAssertionError(
227243
f"General question assertion failed. passed={best.passed}. Reasoning: {best.reasoning}"
228244
)
245+
exc.reasoning_steps = best.reasoning_steps
246+
exc.conversation_id = best.conversation_id
247+
exc.response_id = best.response_id
248+
exc.detail = {
249+
"judge_passed": best.passed,
250+
"judge_reasoning": best.reasoning,
251+
"actual_output": best.actual_output,
252+
}
253+
raise exc
254+
best = summary.best
255+
return AgenticEvalOutcome(
256+
reasoning_steps=best.reasoning_steps,
257+
conversation_id=best.conversation_id,
258+
response_id=best.response_id,
259+
detail={
260+
"judge_passed": best.passed,
261+
"judge_reasoning": best.reasoning,
262+
"actual_output": best.actual_output,
263+
},
264+
)

‎packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py‎

Lines changed: 41 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,11 +3,12 @@
33

44
from __future__ import annotations
55

6-
from dataclasses import dataclass
6+
from dataclasses import dataclass, field
77

88
from gooddata_eval.core.chat.sse_client import ChatClient
99
from gooddata_eval.core.config import ReasoningEffort
1010
from gooddata_eval.core.evaluators._llm_judge import LLMJudge
11+
from gooddata_eval.core.models import AgenticEvalOutcome
1112

1213
_DEFAULT_K = 1
1314

@@ -49,6 +50,8 @@ class GuardrailResult:
4950
passed: bool
5051
llm_judge_score: float
5152
reasoning: str
53+
reasoning_steps: list[str] = field(default_factory=list)
54+
response_id: str | None = None
5255

5356

5457
@dataclass
@@ -95,6 +98,8 @@ def run_agentic_guardrail(
9598
passed=passed,
9699
llm_judge_score=llm_judge_score,
97100
reasoning=reasoning,
101+
reasoning_steps=list(chat_result.reasoning_steps or []),
102+
response_id=chat_result.response_id,
98103
)
99104
)
100105
finally:
@@ -117,6 +122,8 @@ def run_agentic_guardrail(
117122
passed=passed,
118123
llm_judge_score=llm_judge_score,
119124
reasoning=reasoning,
125+
reasoning_steps=list(chat_result.reasoning_steps or []),
126+
response_id=chat_result.response_id,
120127
)
121128
)
122129
finally:
@@ -139,6 +146,10 @@ class GuardrailAssertionError(AssertionError):
139146
"""Raised when a guardrail evaluation fails."""
140147

141148
__tracebackhide__ = True
149+
reasoning_steps: list[str]
150+
conversation_id: str
151+
response_id: str | None
152+
detail: dict
142153

143154

144155
def evaluate_agentic_guardrail(
@@ -157,8 +168,14 @@ def evaluate_agentic_guardrail(
157168
model_version_override: str | None = None,
158169
run_metadata_extra: dict | None = None,
159170
reasoning_effort: ReasoningEffort | None = None,
160-
) -> None:
161-
"""Run guardrail evaluation, log to Langfuse, and raise on failure."""
171+
) -> AgenticEvalOutcome:
172+
"""Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure.
173+
174+
Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
175+
AgenticEvalOutcome on success; on failure the same three values are attached to the
176+
raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors
177+
`evaluate_agentic_metric_skill`'s idiom) so callers can retrieve them either way.
178+
"""
162179
from datetime import datetime as _dt # noqa: PLC0415
163180
from datetime import timezone as _tz # noqa: PLC0415
164181

@@ -220,4 +237,24 @@ def evaluate_agentic_guardrail(
220237

221238
if not summary.pass_at_k:
222239
best = summary.best
223-
raise GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}")
240+
exc = GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}")
241+
exc.reasoning_steps = best.reasoning_steps
242+
exc.conversation_id = best.conversation_id
243+
exc.response_id = best.response_id
244+
exc.detail = {
245+
"judge_passed": best.passed,
246+
"judge_reasoning": best.reasoning,
247+
"actual_output": best.actual_output,
248+
}
249+
raise exc
250+
best = summary.best
251+
return AgenticEvalOutcome(
252+
reasoning_steps=best.reasoning_steps,
253+
conversation_id=best.conversation_id,
254+
response_id=best.response_id,
255+
detail={
256+
"judge_passed": best.passed,
257+
"judge_reasoning": best.reasoning,
258+
"actual_output": best.actual_output,
259+
},
260+
)

0 commit comments

Comments
 (0)