Merge pull request #1763 from gooddata/feat/knowledge-question-evaluator

feat(gooddata-eval): register knowledge_question evaluator

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py

@@ -20,18 +20,26 @@
     )
 }
 
-# LLM-judge evaluators (general_question, guardrail, dashboard_summary) require the
-# [llm-judge] extra. Their modules are imported lazily on first use so the CLI
+# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question)
+# require the [llm-judge] extra. Their modules are imported lazily on first use so the CLI
 # starts without openai.
+#
+# knowledge_question reuses GeneralQuestionEvaluator directly: both are free-text-rubric,
+# LLM-judged prose answers -- knowledge_question just covers platform/product/policy facts
+# instead of LDM-grounded ones. `ItemReport.test_kind` is tagged from the dataset item's own
+# `test_kind` field (see runner.py), not from the evaluator class, so sharing one class
+# across both kinds does not mislabel results.
 _LAZY_EVALUATOR_MODULES: dict[str, str] = {
     "general_question": "gooddata_eval.core.evaluators.general_question",
     "guardrail": "gooddata_eval.core.evaluators.guardrail",
     "dashboard_summary": "gooddata_eval.core.evaluators.summary",
+    "knowledge_question": "gooddata_eval.core.evaluators.general_question",
 }
 _LAZY_EVALUATOR_CLASSES: dict[str, str] = {
     "general_question": "GeneralQuestionEvaluator",
     "guardrail": "GuardrailEvaluator",
     "dashboard_summary": "DashboardSummaryEvaluator",
+    "knowledge_question": "GeneralQuestionEvaluator",
 }
 
 
@@ -57,9 +65,9 @@ def _openai_available() -> bool:
 def supported_test_kinds() -> set[str]:
     """Return all supported test_kind values.
 
-    LLM-judge kinds (general_question, guardrail) are excluded when the
-    [llm-judge] extra (openai) is not installed — those items are skipped
-    rather than erroring out mid-run.
+    LLM-judge kinds (general_question, guardrail, dashboard_summary,
+    knowledge_question) are excluded when the [llm-judge] extra (openai) is
+    not installed — those items are skipped rather than erroring out mid-run.
     """
     kinds = set(_EAGER_EVALUATORS)
     if _openai_available():

packages/gooddata-eval/src/gooddata_eval/core/evaluators/general_question.py

@@ -15,6 +15,10 @@
 
 
 class GeneralQuestionEvaluator:
+    # Also registered under "knowledge_question" (core/evaluators/__init__.py) -- this
+    # attribute is unused by that registration (which keys off item.test_kind, not this
+    # class attribute, see the comment there), so it's harmless as-is, but don't derive
+    # an output label from it without accounting for the second kind it now backs.
     test_kind = "general_question"
 
     def __init__(self):

packages/gooddata-eval/tests/test_runner.py

@@ -122,6 +122,7 @@ def test_run_items_routes_all_supported_kinds():
         "general_question",
         "guardrail",
         "dashboard_summary",
+        "knowledge_question",
     }
     assert expected_kinds == supported_test_kinds()
 

packages/gooddata-eval/tests/test_text_evaluators.py

@@ -1,6 +1,7 @@
 # (C) 2026 GoodData Corporation
 from unittest.mock import MagicMock, patch
 
+from gooddata_eval.core.evaluators import get_evaluator
 from gooddata_eval.core.evaluators._llm_judge import JudgeResponseError
 from gooddata_eval.core.evaluators.general_question import GeneralQuestionEvaluator
 from gooddata_eval.core.evaluators.guardrail import GuardrailEvaluator
@@ -17,6 +18,16 @@ def _gq_item() -> DatasetItem:
     )
 
 
+def _kq_item() -> DatasetItem:
+    return DatasetItem(
+        id="kq-001",
+        dataset_name="d",
+        test_kind="knowledge_question",
+        question="How can I get access to more ICAs?",
+        expected_output="Must explain the process for requesting additional ICAs, e.g. contacting an account manager.",
+    )
+
+
 def _gr_item() -> DatasetItem:
     return DatasetItem(
         id="gr-001",
@@ -60,6 +71,23 @@ def test_general_question_fails_when_judge_scores_0():
     assert result.passed is False
 
 
+def test_knowledge_question_dispatches_to_general_question_evaluator():
+    with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(True)):
+        assert isinstance(get_evaluator("knowledge_question"), GeneralQuestionEvaluator)
+
+
+def test_knowledge_question_passes_when_judge_scores_1():
+    with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(True)):
+        result = GeneralQuestionEvaluator().evaluate(_kq_item(), _chat_text("Contact your account manager."))
+    assert result.passed is True
+
+
+def test_knowledge_question_fails_when_judge_scores_0():
+    with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(False)):
+        result = GeneralQuestionEvaluator().evaluate(_kq_item(), _chat_text("I don't know."))
+    assert result.passed is False
+
+
 def test_guardrail_fails_when_viz_returned():
     with patch("gooddata_eval.core.evaluators.guardrail.LLMJudge", return_value=_make_judge(True)):
         result = GuardrailEvaluator().evaluate(_gr_item(), _chat_viz())