diff --git a/pyrit/score/fallback_scorer.py b/pyrit/score/fallback_scorer.py index dbf8e468f4..4f5848d8e1 100644 --- a/pyrit/score/fallback_scorer.py +++ b/pyrit/score/fallback_scorer.py @@ -6,6 +6,7 @@ from pyrit.models import ComponentIdentifier, MessageScorable, Scorable, Score, ScoringExpectation from pyrit.score.observation.execution import _merge_observation_ids +from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY from pyrit.score.scorer import Scorer if TYPE_CHECKING: @@ -139,6 +140,9 @@ def _build_result( f"primary.{key}": value for key, value in (primary.score_metadata or {}).items() } metadata["resolved_by"] = "fallback" if fallback is not None else "primary" + # Attacks rank true/false scores by this top-level key, so keep the selected child's value. + if selected.score_metadata and ORIGINAL_FLOAT_VALUE_KEY in selected.score_metadata: + metadata[ORIGINAL_FLOAT_VALUE_KEY] = selected.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] rationale = primary.score_rationale if fallback_attempted: metadata["primary_rationale"] = primary.score_rationale or "" diff --git a/tests/unit/score/test_fallback_scorer.py b/tests/unit/score/test_fallback_scorer.py index 86b8d77ab3..c6d8efc1a0 100644 --- a/tests/unit/score/test_fallback_scorer.py +++ b/tests/unit/score/test_fallback_scorer.py @@ -40,6 +40,7 @@ TrueFalseScorer, ) from pyrit.score.fallback_scorer import _FallbackScorer +from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY, normalize_score_to_float pytestmark = pytest.mark.usefixtures("patch_central_database") @@ -156,6 +157,48 @@ async def test_complete_primary_skips_fallback_async( assert isinstance(wrapper, FloatScaleScorer if family == "float_scale" else TrueFalseScorer) +@pytest.mark.parametrize("primary_resolves", [False, True]) +@pytest.mark.parametrize("children_report_floats", [False, True]) +async def test_selected_original_float_value_stays_readable_async( + *, primary_resolves: bool, children_report_floats: bool, sqlite_instance: MemoryInterface +) -> None: + primary, fallback = _TrueFalseScorer(), _TrueFalseScorer() + wrapper = TrueFalseFallbackScorer(scorer=primary, fallback_scorer=fallback) + primary_score = _score( + family="true_false", + value="True" if primary_resolves else None, + rationale="primary reason", + metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.3} if children_report_floats else {"model": "primary"}, + ) + fallback_score = _score( + family="true_false", + value="True", + rationale="fallback reason", + metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.6} if children_report_floats else {"model": "fallback"}, + ) + selected = primary_score if primary_resolves else fallback_score + with ( + patch.object(primary, "_score_scorable_async", new_callable=AsyncMock, return_value=[primary_score]), + patch.object(fallback, "_score_scorable_async", new_callable=AsyncMock, return_value=[fallback_score]), + ): + result = (await wrapper.score_async(scorable=ContentScorable(value="evidence")))[0] + [persisted] = await sqlite_instance.get_scores_async(score_ids=[str(result.id)]) + + for score in (result, persisted): + assert score.get_value() is True + assert score.score_category == ["criterion"] + assert score.score_metadata["resolved_by"] == ("primary" if primary_resolves else "fallback") + assert score.score_metadata.get("primary.original_float_value") == (0.3 if children_report_floats else None) + if children_report_floats: + assert score.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] == selected.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] + assert normalize_score_to_float(score) == (0.3 if primary_resolves else 0.6) + else: + assert ORIGINAL_FLOAT_VALUE_KEY not in score.score_metadata + assert normalize_score_to_float(score) == 1.0 + assert result.score_rationale == persisted.score_rationale + assert ("fallback reason" in result.score_rationale) is not primary_resolves + + @pytest.mark.parametrize("fallback_abstains", [False, True]) async def test_fallback_preserves_both_attempts_without_mutating_children_async( *, pair: tuple[Scorer, Scorer, Scorer], family: ScoreType, fallback_abstains: bool, sqlite_instance: MemoryInterface