fix(gooddata-eval): reject malformed alert group-by arguments (GDAI-2175)

`actual_args.get("attributes") or []` collapsed a falsy non-list -- `{}`,
`""`, `0` -- into "no grouping", so those passed an `Attributes: []`
expectation. `_attribute_label_ids` also never checked that its input was a
list, so a truthy non-list was iterated instead: a dict yields its keys, a
string yields its characters. That produced a false pass, an actual of
`"x"` scoring correct against an expectation of `[{"using": "label/x"}]`.

The two malformations are not the same kind of event, and the split is
structural rather than by side:

- Not a list of groupings at all -> False. The agent answered wrongly, and a
  wrong answer is a FAIL. Raising would record an ERROR, which
  `json_report.py` excludes from the failure count, so a malformed answer
  would rank above a merely wrong one.
- A list holding an unreadable entry -> raises, as before. Entries are typed
  `AttributeItem` at the tool boundary, so the plausible cause is the wire
  format moving, and that has to be unmissable rather than read as every
  agent regressing at once.

The expectation side needs no equivalent guard: `_normalize_expected_output`
already rejects a non-list before the run starts, and `from_dict`, the other
way into the field, has no callers.

risk: low

Author: tychtjan

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -140,6 +140,8 @@ def _attribute_label_ids(items: list, *, side: str) -> list[str]:
     ``attribute/x`` are different objects, and an unknown spelling must fail loudly rather
     than quietly compare unequal.
     """
+    if not isinstance(items, list):
+        raise ValueError(f"Unrecognised {side} group-by attributes, expected a list: {items!r}")
     ids: list[str] = []
     for item in items:
         raw: object = None
@@ -174,9 +176,20 @@ def _check_attributes(expected: CatalogMetricAlert, actual_args: dict) -> bool:
     matter.
     """
     exp_attributes = expected.attributes
-    act_attributes = actual_args.get("attributes") or []
     if exp_attributes is None:
         return True
+    # An unset argument serialises as null, so both spellings of "no grouping" land on [].
+    act_attributes = actual_args.get("attributes")
+    if act_attributes is None:
+        act_attributes = []
+    if not isinstance(act_attributes, list):
+        # An argument that is not a list of groupings is the agent answering wrongly, so it
+        # scores False. Raising instead would make the runner record an ERROR, and errored
+        # items are excluded from the failure count — a malformed answer must not rank above
+        # a merely wrong one. An unreadable *entry* still raises, in `_attribute_label_ids`:
+        # entries are typed at the tool boundary, so the plausible cause there is the wire
+        # format moving, which has to be unmissable.
+        return False
     if not exp_attributes:
         return not act_attributes
     exp_ids = sorted(_attribute_label_ids(exp_attributes, side="expected"))

packages/gooddata-eval/tests/test_agentic_alert_skill.py

@@ -843,6 +843,21 @@ def test_unrecognised_expectation_shape_raises():
         _normalize_expected_output({"Attributes": [{"using": "attribute/product_brand"}]})
 
 
+def test_malformed_actual_attributes_fail_rather_than_error():
+    """A non-list argument is the agent answering wrongly, and a wrong answer is a FAIL.
+
+    An ERROR would be excluded from the run's failure count, ranking a malformed answer
+    above a merely wrong one.
+    """
+    expected_none = _normalize_expected_output({"Attributes": []})
+    for malformed in ({}, "", 0, {"using": "label/product_brand"}, "product_brand"):
+        assert _check_attributes(expected_none, {"attributes": malformed}) is False
+
+    expected_brand = _normalize_expected_output({"Attributes": [_AAC_BRAND_GROUPING]})
+    for malformed in ({}, "", 0, "product_brand", {"using": "label/product_brand"}):
+        assert _check_attributes(expected_brand, {"attributes": malformed}) is False
+
+
 def test_unrecognised_actual_shape_raises():
     expected = _normalize_expected_output({"Attributes": [_AAC_BRAND_GROUPING]})
     with pytest.raises(ValueError, match="actual group-by"):