FIX: threshold scorer attributes result to first score, not the aggregate

pyrit/score/score_utils.py

@@ -17,14 +17,22 @@ def combine_metadata_and_categories(scores: list[Score]) -> tuple[dict[str, str
         scores: List of Score objects.
 
     Returns:
-        Tuple of (metadata dict, sorted category list with empty strings filtered).
+        Tuple of (unambiguous metadata dict, sorted category list with empty strings filtered).
     """
     metadata: dict[str, str | int | float] = {}
+    conflicting_metadata_keys: set[str] = set()
     category_set: set[str] = set()
 
     for s in scores:
         if s.score_metadata:
-            metadata.update(s.score_metadata)
+            for key, value in s.score_metadata.items():
+                if key in conflicting_metadata_keys:
+                    continue
+                if key in metadata and metadata[key] != value:
+                    metadata.pop(key)
+                    conflicting_metadata_keys.add(key)
+                    continue
+                metadata[key] = value
         score_categories = s.score_category or []
         category_set.update([c for c in score_categories if c])
 

pyrit/score/true_false/float_scale_threshold_scorer.py

@@ -147,18 +147,26 @@ async def _score_prepared_message_async(
             score = scores[0]
             score.score_type = "true_false"
             score.score_value = str(threshold_result)
+            # Carry the aggregate's category, metadata and rationale rather than the first
+            # constituent score's. The threshold decision is made on the aggregate, so
+            # describing it with scores[0] mislabels the result whenever the wrapped scorer
+            # returns more than one score (e.g. AzureContentFilterScorer, one per harm
+            # category): the value would say True while the category, rationale and metadata
+            # described a different, possibly zero-valued, category.
             score.score_rationale = (
                 f"based on {scorer_type}\n"
                 f"Normalized scale score: {aggregate_value} {comparison_symbol} threshold {self._threshold}\n"
-                f"Rationale for scale score: {score.score_rationale}"
+                f"Rationale for scale score: {aggregate_score.rationale}"
             )
             score.score_value_description = aggregate_score.description
+            score.score_category = aggregate_score.category
             score.id = uuid.uuid4()
             score.scorer_class_identifier = self.get_identifier()
             # Store the original float value in metadata for granular comparison
-            if score.score_metadata is None:
-                score.score_metadata = {}
-            score.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] = aggregate_value
+            score.score_metadata = {
+                **aggregate_score.metadata,
+                ORIGINAL_FLOAT_VALUE_KEY: aggregate_value,
+            }
         else:
             # Create new score from aggregator result (all pieces were filtered out)
             # Use the first message piece's id if available, otherwise generate a new UUID

tests/unit/score/test_float_scale_threshold_scorer.py

@@ -133,6 +133,83 @@ async def test_float_scale_threshold_scorer_returns_single_score_with_multi_cate
         assert len(added_scores) == 1
 
 
+async def test_float_scale_threshold_scorer_attributes_result_to_aggregate_not_first_score():
+    """
+    The threshold decision is made on the aggregate, so the resulting score must be described
+    by the aggregate too. Previously the category, rationale and metadata were taken from
+    scores[0], so a scorer returning one score per harm category (AzureContentFilterScorer)
+    produced a True score labelled with whichever category happened to be first, even when
+    that category scored 0.0.
+    """
+
+    memory = MagicMock(MemoryInterface)
+    mock_identifier = ComponentIdentifier(class_name="MockScorer", class_module="test.mock")
+
+    prompt_id = uuid.uuid4()
+    scorer = MagicMock(spec=FloatScaleScorer)
+    scorer._score_nested_message_async = AsyncMock(
+        return_value=[
+            Score(
+                score_value="0.857",
+                score_type="float_scale",
+                score_category=["Violence"],
+                score_rationale="Violence rationale",
+                score_metadata={"azure_severity": 6},
+                message_piece_id=prompt_id,
+                score_value_description="",
+                scorer_class_identifier=mock_identifier,
+                id=uuid.uuid4(),
+            ),
+            Score(
+                score_value="0.0",
+                score_type="float_scale",
+                score_category=["Hate"],
+                score_rationale="Hate rationale",
+                score_metadata={"azure_severity": 0},
+                message_piece_id=prompt_id,
+                score_value_description="",
+                scorer_class_identifier=mock_identifier,
+                id=uuid.uuid4(),
+            ),
+        ]
+    )
+    scorer.get_identifier = MagicMock(return_value=mock_identifier)
+
+    with patch.object(CentralMemory, "get_memory_instance", return_value=memory):
+        threshold_scorer = FloatScaleThresholdScorer(scorer=scorer, threshold=0.5)
+        score = (await threshold_scorer.score_text_async(text="mock example"))[0]
+
+    # Violence (0.857) is what crossed the threshold; Hate scored 0.0.
+    assert score.get_value() is True
+
+    # The category that fired must not be dropped in favour of the first score's.
+    assert "Violence" in (score.score_category or [])
+
+    # The rationale must mention the score that actually crossed, not only the first one.
+    assert "Violence rationale" in score.score_rationale
+
+    # The aggregate spans categories with different severities, so the ambiguous
+    # category-specific severity must not be paired with the aggregate value.
+    assert score.score_metadata["original_float_value"] == pytest.approx(0.857)
+    assert "azure_severity" not in score.score_metadata
+
+
+async def test_float_scale_threshold_scorer_single_score_attribution_unchanged():
+    """A single wrapped score must keep its own category and rationale, as before."""
+
+    memory = MagicMock(MemoryInterface)
+    scorer = create_mock_float_scorer(0.9)
+
+    with patch.object(CentralMemory, "get_memory_instance", return_value=memory):
+        threshold_scorer = FloatScaleThresholdScorer(scorer=scorer, threshold=0.5)
+        score = (await threshold_scorer.score_text_async(text="mock example"))[0]
+
+    assert score.get_value() is True
+    assert score.score_category == ["mock category"]
+    assert "A mock rationale" in score.score_rationale
+    assert score.score_metadata["original_float_value"] == pytest.approx(0.9)
+
+
 async def test_float_scale_threshold_scorer_handles_empty_scores():
     """
     Test that FloatScaleThresholdScorer gracefully handles when the underlying scorer

tests/unit/score/test_score_utils.py

@@ -194,6 +194,22 @@ def test_combines_metadata_from_multiple_scores(self) -> None:
         assert metadata == {"key1": "value1", "key2": "value2"}
         assert categories == ["cat1", "cat2"]
 
+    def test_omits_conflicting_metadata(self) -> None:
+        """Metadata with different values must not depend on score order."""
+        score1 = MagicMock()
+        score1.score_metadata = {"shared": "first", "same": 1}
+        score1.score_category = []
+
+        score2 = MagicMock()
+        score2.score_metadata = {"shared": "second", "same": 1}
+        score2.score_category = []
+
+        metadata, _ = combine_metadata_and_categories([score1, score2])
+        reversed_metadata, _ = combine_metadata_and_categories([score2, score1])
+
+        assert metadata == {"same": 1}
+        assert reversed_metadata == metadata
+
     def test_deduplicates_categories(self) -> None:
         """Duplicate categories should be removed."""
         score1 = MagicMock()