Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions pyrit/score/fallback_scorer.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

from pyrit.models import ComponentIdentifier, MessageScorable, Scorable, Score, ScoringExpectation
from pyrit.score.observation.execution import _merge_observation_ids
from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY
from pyrit.score.scorer import Scorer

if TYPE_CHECKING:
Expand Down Expand Up @@ -139,6 +140,9 @@ def _build_result(
f"primary.{key}": value for key, value in (primary.score_metadata or {}).items()
}
metadata["resolved_by"] = "fallback" if fallback is not None else "primary"
# Attacks rank true/false scores by this top-level key, so keep the selected child's value.
if selected.score_metadata and ORIGINAL_FLOAT_VALUE_KEY in selected.score_metadata:
metadata[ORIGINAL_FLOAT_VALUE_KEY] = selected.score_metadata[ORIGINAL_FLOAT_VALUE_KEY]
rationale = primary.score_rationale
if fallback_attempted:
metadata["primary_rationale"] = primary.score_rationale or ""
Expand Down
43 changes: 43 additions & 0 deletions tests/unit/score/test_fallback_scorer.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
TrueFalseScorer,
)
from pyrit.score.fallback_scorer import _FallbackScorer
from pyrit.score.score_utils import ORIGINAL_FLOAT_VALUE_KEY, normalize_score_to_float

pytestmark = pytest.mark.usefixtures("patch_central_database")

Expand Down Expand Up @@ -156,6 +157,48 @@ async def test_complete_primary_skips_fallback_async(
assert isinstance(wrapper, FloatScaleScorer if family == "float_scale" else TrueFalseScorer)


@pytest.mark.parametrize("primary_resolves", [False, True])
@pytest.mark.parametrize("children_report_floats", [False, True])
async def test_selected_original_float_value_stays_readable_async(
*, primary_resolves: bool, children_report_floats: bool, sqlite_instance: MemoryInterface
) -> None:
primary, fallback = _TrueFalseScorer(), _TrueFalseScorer()
wrapper = TrueFalseFallbackScorer(scorer=primary, fallback_scorer=fallback)
primary_score = _score(
family="true_false",
value="True" if primary_resolves else None,
rationale="primary reason",
metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.3} if children_report_floats else {"model": "primary"},
)
fallback_score = _score(
family="true_false",
value="True",
rationale="fallback reason",
metadata={ORIGINAL_FLOAT_VALUE_KEY: 0.6} if children_report_floats else {"model": "fallback"},
)
selected = primary_score if primary_resolves else fallback_score
with (
patch.object(primary, "_score_scorable_async", new_callable=AsyncMock, return_value=[primary_score]),
patch.object(fallback, "_score_scorable_async", new_callable=AsyncMock, return_value=[fallback_score]),
):
result = (await wrapper.score_async(scorable=ContentScorable(value="evidence")))[0]
[persisted] = await sqlite_instance.get_scores_async(score_ids=[str(result.id)])

for score in (result, persisted):
assert score.get_value() is True
assert score.score_category == ["criterion"]
assert score.score_metadata["resolved_by"] == ("primary" if primary_resolves else "fallback")
assert score.score_metadata.get("primary.original_float_value") == (0.3 if children_report_floats else None)
if children_report_floats:
assert score.score_metadata[ORIGINAL_FLOAT_VALUE_KEY] == selected.score_metadata[ORIGINAL_FLOAT_VALUE_KEY]
assert normalize_score_to_float(score) == (0.3 if primary_resolves else 0.6)
else:
assert ORIGINAL_FLOAT_VALUE_KEY not in score.score_metadata
assert normalize_score_to_float(score) == 1.0
assert result.score_rationale == persisted.score_rationale
assert ("fallback reason" in result.score_rationale) is not primary_resolves


@pytest.mark.parametrize("fallback_abstains", [False, True])
async def test_fallback_preserves_both_attempts_without_mutating_children_async(
*, pair: tuple[Scorer, Scorer, Scorer], family: ScoreType, fallback_abstains: bool, sqlite_instance: MemoryInterface
Expand Down
Loading