From 8d52198e6419e853a01e0c329bd54edf47fa5bcb Mon Sep 17 00:00:00 2001 From: Utkarsh Bahuguna Date: Sun, 11 Oct 2026 00:38:10 +0530 Subject: [PATCH 1/2] keep original_float_value on fallback scores --- pyrit/score/fallback_scorer.py | 4 ++++ tests/unit/score/test_fallback_scorer.py | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/pyrit/score/fallback_scorer.py b/pyrit/score/fallback_scorer.py index dbf8e468f4..4f5848d8e1 100644 --- a/pyrit/score/fallback_scorer.py +++ b/pyrit/score/fallback_scorer.py @@ -6,6 +6,7 @@ from pyrit.models import ComponentIdentifier, MessageScorable, Scorable, Score, ScoringExpectation from pyrit.score.observation.execution import _merge_observation_ids +from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY from pyrit.score.scorer import Scorer if TYPE_CHECKING: @@ -139,6 +140,9 @@ def _build_result( f"primary.{key}": value for key, value in (primary.score_metadata or {}).items() } metadata["resolved_by"] = "fallback" if fallback is not None else "primary" + # Attacks rank true/false scores by this top-level key, so keep the selected child's value. + if selected.score_metadata and ORIGINAL_FLOAT_VALUE_KEY in selected.score_metadata: + metadata[ORIGINAL_FLOAT_VALUE_KEY] = selected.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] rationale = primary.score_rationale if fallback_attempted: metadata["primary_rationale"] = primary.score_rationale or "" diff --git a/tests/unit/score/test_fallback_scorer.py b/tests/unit/score/test_fallback_scorer.py index 86b8d77ab3..c55d69aea8 100644 --- a/tests/unit/score/test_fallback_scorer.py +++ b/tests/unit/score/test_fallback_scorer.py @@ -40,6 +40,7 @@ TrueFalseScorer, ) from pyrit.score.fallback_scorer import _FallbackScorer +from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY, normalize_score_to_float pytestmark = pytest.mark.usefixtures("patch_central_database") @@ -156,6 +157,25 @@ async def test_complete_primary_skips_fallback_async( assert isinstance(wrapper, FloatScaleScorer if family == "float_scale" else TrueFalseScorer) +@pytest.mark.parametrize("primary_resolves", [False, True]) +async def test_selected_original_float_value_stays_readable_async(*, primary_resolves: bool) -> None: + primary, fallback = _TrueFalseScorer(), _TrueFalseScorer() + wrapper = TrueFalseFallbackScorer(scorer=primary, fallback_scorer=fallback) + primary_score = _score( + family="true_false", + value="True" if primary_resolves else None, + metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.3}, + ) + fallback_score = _score(family="true_false", value="True", metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.6}) + with ( + patch.object(primary, "_score_scorable_async", new_callable=AsyncMock, return_value=[primary_score]), + patch.object(fallback, "_score_scorable_async", new_callable=AsyncMock, return_value=[fallback_score]), + ): + result = (await wrapper.score_async(scorable=ContentScorable(value="evidence")))[0] + + assert normalize_score_to_float(result) == (0.3 if primary_resolves else 0.6) + + @pytest.mark.parametrize("fallback_abstains", [False, True]) async def test_fallback_preserves_both_attempts_without_mutating_children_async( *, pair: tuple[Scorer, Scorer, Scorer], family: ScoreType, fallback_abstains: bool, sqlite_instance: MemoryInterface From 64fcfa00325cdebb3b521bd3874443fd936d9eef Mon Sep 17 00:00:00 2001 From: Utkarsh Bahuguna Date: Sun, 11 Oct 2026 10:43:13 +0530 Subject: [PATCH 2/2] tighten tests: async names, typed args, public paths --- tests/unit/score/test_fallback_scorer.py | 33 ++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/tests/unit/score/test_fallback_scorer.py b/tests/unit/score/test_fallback_scorer.py index c55d69aea8..c6d8efc1a0 100644 --- a/tests/unit/score/test_fallback_scorer.py +++ b/tests/unit/score/test_fallback_scorer.py @@ -158,22 +158,45 @@ async def test_complete_primary_skips_fallback_async( @pytest.mark.parametrize("primary_resolves", [False, True]) -async def test_selected_original_float_value_stays_readable_async(*, primary_resolves: bool) -> None: +@pytest.mark.parametrize("children_report_floats", [False, True]) +async def test_selected_original_float_value_stays_readable_async( + *, primary_resolves: bool, children_report_floats: bool, sqlite_instance: MemoryInterface +) -> None: primary, fallback = _TrueFalseScorer(), _TrueFalseScorer() wrapper = TrueFalseFallbackScorer(scorer=primary, fallback_scorer=fallback) primary_score = _score( family="true_false", value="True" if primary_resolves else None, - metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.3}, + rationale="primary reason", + metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.3} if children_report_floats else {"model": "primary"}, + ) + fallback_score = _score( + family="true_false", + value="True", + rationale="fallback reason", + metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.6} if children_report_floats else {"model": "fallback"}, ) - fallback_score = _score(family="true_false", value="True", metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.6}) + selected = primary_score if primary_resolves else fallback_score with ( patch.object(primary, "_score_scorable_async", new_callable=AsyncMock, return_value=[primary_score]), patch.object(fallback, "_score_scorable_async", new_callable=AsyncMock, return_value=[fallback_score]), ): result = (await wrapper.score_async(scorable=ContentScorable(value="evidence")))[0] - - assert normalize_score_to_float(result) == (0.3 if primary_resolves else 0.6) + [persisted] = await sqlite_instance.get_scores_async(score_ids=[str(result.id)]) + + for score in (result, persisted): + assert score.get_value() is True + assert score.score_category == ["criterion"] + assert score.score_metadata["resolved_by"] == ("primary" if primary_resolves else "fallback") + assert score.score_metadata.get("primary.original_float_value") == (0.3 if children_report_floats else None) + if children_report_floats: + assert score.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] == selected.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] + assert normalize_score_to_float(score) == (0.3 if primary_resolves else 0.6) + else: + assert ORIGINAL_FLOAT_VALUE_KEY not in score.score_metadata + assert normalize_score_to_float(score) == 1.0 + assert result.score_rationale == persisted.score_rationale + assert ("fallback reason" in result.score_rationale) is not primary_resolves @pytest.mark.parametrize("fallback_abstains", [False, True])