From cf8d2ff772cb454db58bbe74b90e4ad577aa4e1f Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sat, 22 Aug 2026 16:24:10 +0800 Subject: [PATCH 1/4] FIX: reject non-finite score_value in numeric response handler JsonSchemaResponseHandler with numeric_value=True validated the value with a bare float() cast, so "nan", "inf" and "-inf" strings passed as valid scores. Float-scale aggregation then clamps with max(0.0, min(1.0, x)); since every comparison against NaN is False, min(1.0, nan) returns 1.0 and a NaN judge response silently aggregates toward the maximum harm score instead of surfacing an error. Treat non-finite parsed values as invalid scoring responses via math.isfinite. Regression tests cover the five accepted spellings. Fixes #2458 Signed-off-by: fei <204683769+feiiiiii5@users.noreply.github.com> --- pyrit/score/response_handler.py | 7 ++++++- tests/unit/score/test_response_handler.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pyrit/score/response_handler.py b/pyrit/score/response_handler.py index 6dea0a9d0a..63ae53a785 100644 --- a/pyrit/score/response_handler.py +++ b/pyrit/score/response_handler.py @@ -5,6 +5,7 @@ import abc import json +import math from abc import abstractmethod from collections.abc import Sequence from typing import TYPE_CHECKING @@ -251,11 +252,15 @@ def parse( try: # A numeric handler requires the score value to be parsable as a float; a # well-formed-but-non-numeric value is treated as an invalid response. - float(score.raw_score_value) + parsed_value = float(score.raw_score_value) except ValueError: raise InvalidJsonException( message=f"Invalid JSON response, score_value should be a float not this: {score.raw_score_value}" ) from None + if not math.isfinite(parsed_value): + raise InvalidJsonException( + message=f"Invalid JSON response, score_value must be a finite float: {score.raw_score_value}" + ) return score diff --git a/tests/unit/score/test_response_handler.py b/tests/unit/score/test_response_handler.py index 16084a5cf6..ad91e4278d 100644 --- a/tests/unit/score/test_response_handler.py +++ b/tests/unit/score/test_response_handler.py @@ -22,6 +22,18 @@ def test_json_schema_response_handler_rejects_non_object_response(response_text: ) +@pytest.mark.parametrize("score_value", ["nan", "NaN", "inf", "-inf", "Infinity"]) +def test_json_schema_response_handler_rejects_non_finite_numeric_values(score_value: str) -> None: + handler = JsonSchemaResponseHandler(numeric_value=True) + + with pytest.raises(InvalidJsonException, match="finite float"): + handler.parse( + response_text=f'{{"score_value": "{score_value}", "rationale": "test"}}', + scorer_identifier=SCORER_IDENTIFIER, + scored_prompt_id="test-id", + ) + + @pytest.mark.parametrize( ("json_value", "expected"), [ From a79b48677b47c5c98bd30aedbfd4b140332042bb Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:27:30 +0800 Subject: [PATCH 2/4] fix(frontend): surface auth config fetch failures instead of treating them as auth-disabled fetchAuthConfig collapsed both non-2xx responses and network errors into an empty AuthConfig, which AuthProvider interpreted as "authentication disabled". When /api/auth/config failed transiently while protected APIs returned 401, the app rendered the normal shell with raw "Missing or invalid Authorization header" text and no login control or error surface (#2441). The two failure paths now throw so the existing AuthProvider catch renders its Authentication Error page. A 200 response with an empty config still means auth is disabled (local dev), unchanged. Fixes #2441 Signed-off-by: fei <204683769+feiiiiii5@users.noreply.github.com> --- frontend/src/auth/msalConfig.test.ts | 16 +++++++++------- frontend/src/auth/msalConfig.ts | 22 +++++++++++++--------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/frontend/src/auth/msalConfig.test.ts b/frontend/src/auth/msalConfig.test.ts index bda21bf5fa..2ed9472849 100644 --- a/frontend/src/auth/msalConfig.test.ts +++ b/frontend/src/auth/msalConfig.test.ts @@ -73,22 +73,24 @@ describe("msalConfig", () => { expect(global.fetch).toHaveBeenCalledWith("/api/auth/config"); }); - it("returns empty config when response is not ok", async () => { - (global.fetch as jest.Mock).mockResolvedValue({ ok: false }); + it("throws when response is not ok (transient failure is not auth-disabled)", async () => { + (global.fetch as jest.Mock).mockResolvedValue({ + ok: false, + status: 503, + statusText: "Service Unavailable", + }); const { fetchAuthConfig } = await import("./msalConfig"); - const result = await fetchAuthConfig(); - expect(result).toEqual({ clientId: "", tenantId: "", allowedGroupIds: "" }); + await expect(fetchAuthConfig()).rejects.toThrow("/api/auth/config returned 503 Service Unavailable"); }); - it("returns empty config on network error", async () => { + it("throws on network error (transient failure is not auth-disabled)", async () => { (global.fetch as jest.Mock).mockRejectedValue(new Error("Network error")); const { fetchAuthConfig } = await import("./msalConfig"); - const result = await fetchAuthConfig(); - expect(result).toEqual({ clientId: "", tenantId: "", allowedGroupIds: "" }); + await expect(fetchAuthConfig()).rejects.toThrow("Failed to reach /api/auth/config"); }); }); }); diff --git a/frontend/src/auth/msalConfig.ts b/frontend/src/auth/msalConfig.ts index c63e3a68cf..8addef4f72 100644 --- a/frontend/src/auth/msalConfig.ts +++ b/frontend/src/auth/msalConfig.ts @@ -24,17 +24,21 @@ export interface AuthConfig { } export async function fetchAuthConfig(): Promise { + let response: Response try { - const response = await fetch('/api/auth/config') - if (!response.ok) { - // Auth endpoint not available — treat as auth disabled - return { clientId: '', tenantId: '', allowedGroupIds: '' } - } - return (await response.json()) as AuthConfig - } catch { - // Network error (e.g., backend not running yet) — treat as auth disabled - return { clientId: '', tenantId: '', allowedGroupIds: '' } + response = await fetch('/api/auth/config') + } catch (e) { + // A network error (e.g., backend not running yet) is a transient + // infrastructure failure, not proof that auth is disabled. Surface it so + // AuthProvider can show its error state instead of rendering the shell + // while protected APIs return 401. + throw new Error(`Failed to reach /api/auth/config: ${e instanceof Error ? e.message : String(e)}`) } + if (!response.ok) { + // HTTP-level failures on the config endpoint are equally inconclusive. + throw new Error(`/api/auth/config returned ${response.status} ${response.statusText}`) + } + return (await response.json()) as AuthConfig } export function buildMsalConfig(authConfig: AuthConfig): Configuration { From ac1fc88337ba570bf4feb70f5ef2f04da7783a91 Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sat, 22 Aug 2026 20:52:56 +0800 Subject: [PATCH 3/4] fix(scenario): warn when selected techniques have no registered factory resolve_technique_factories silently dropped selected techniques that have no registered factory, so a scenario selecting ["a", "b", "c"] where "b" is unregistered ran with 2 of 3 techniques and no indication the selection was partially ignored (#2461). Log a warning listing the dropped selections, matching how adaptive_scenario.py already surfaces skipped techniques. Fixes #2461 Signed-off-by: fei <204683769+feiiiiii5@users.noreply.github.com> --- pyrit/scenario/core/matrix_atomic_attack_builder.py | 9 +++++++++ .../core/test_matrix_atomic_attack_builder.py | 13 +++++++++++++ 2 files changed, 22 insertions(+) diff --git a/pyrit/scenario/core/matrix_atomic_attack_builder.py b/pyrit/scenario/core/matrix_atomic_attack_builder.py index 40a35ca474..fc7f8e26a3 100644 --- a/pyrit/scenario/core/matrix_atomic_attack_builder.py +++ b/pyrit/scenario/core/matrix_atomic_attack_builder.py @@ -160,6 +160,15 @@ def resolve_technique_factories( all_factories = dict(AttackTechniqueRegistry.get_registry_singleton().get_factories_or_raise()) if extra_factories: all_factories.update(extra_factories) + missing = [ + technique.value + for technique in context.scenario_techniques + if technique.value not in all_factories + ] + if missing: + logger.warning( + f"Selected techniques have no registered factory and will be skipped: {missing}" + ) return { technique.value: all_factories[technique.value] for technique in context.scenario_techniques diff --git a/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py b/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py index 044ce78512..8b75c9174d 100644 --- a/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py +++ b/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py @@ -18,6 +18,8 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch +import logging + import pytest from pyrit.models import AttackSeedGroup, SeedObjective @@ -408,6 +410,17 @@ def test_drops_techniques_without_factory(self): resolved = resolve_technique_factories(context=context) assert list(resolved.keys()) == ["alpha"] + def test_warns_when_selected_technique_has_no_factory(self, caplog, ): + factories = {"alpha": _mock_factory(name="alpha")} + context = _context(techniques=[_technique("alpha"), _technique("missing")]) + with ( + _patch_registry(factories), + caplog.at_level(logging.WARNING, logger="pyrit.scenario.core.matrix_atomic_attack_builder"), + ): + resolved = resolve_technique_factories(context=context) + assert list(resolved.keys()) == ["alpha"] + assert any("missing" in r.getMessage() for r in caplog.records) + def test_extra_factories_merged_and_override_registry(self): registry_factories = {"alpha": _mock_factory(name="alpha")} local_alpha = _mock_factory(name="alpha") From 52ae9785de6619d51eef2443fa1cb99e5ba7eb91 Mon Sep 17 00:00:00 2001 From: Roman Lutz Date: Sat, 22 Aug 2026 06:53:03 -0700 Subject: [PATCH 4/4] fix: address self-review findings Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- frontend/src/auth/msalConfig.ts | 4 +++- pyrit/scenario/core/matrix_atomic_attack_builder.py | 12 +++--------- pyrit/score/response_handler.py | 2 +- .../core/test_matrix_atomic_attack_builder.py | 5 ++--- 4 files changed, 9 insertions(+), 14 deletions(-) diff --git a/frontend/src/auth/msalConfig.ts b/frontend/src/auth/msalConfig.ts index 8addef4f72..1d4a70b121 100644 --- a/frontend/src/auth/msalConfig.ts +++ b/frontend/src/auth/msalConfig.ts @@ -32,7 +32,9 @@ export async function fetchAuthConfig(): Promise { // infrastructure failure, not proof that auth is disabled. Surface it so // AuthProvider can show its error state instead of rendering the shell // while protected APIs return 401. - throw new Error(`Failed to reach /api/auth/config: ${e instanceof Error ? e.message : String(e)}`) + throw Object.assign(new Error(`Failed to reach /api/auth/config: ${e instanceof Error ? e.message : String(e)}`), { + cause: e, + }) } if (!response.ok) { // HTTP-level failures on the config endpoint are equally inconclusive. diff --git a/pyrit/scenario/core/matrix_atomic_attack_builder.py b/pyrit/scenario/core/matrix_atomic_attack_builder.py index fc7f8e26a3..34787b248b 100644 --- a/pyrit/scenario/core/matrix_atomic_attack_builder.py +++ b/pyrit/scenario/core/matrix_atomic_attack_builder.py @@ -142,7 +142,7 @@ def resolve_technique_factories( Reads the ``AttackTechniqueRegistry`` singleton and keeps only the factories whose name matches a selected technique, preserving selection order. Techniques with no registered - factory are silently dropped so the caller can proceed with whatever techniques exist. + factory are logged and dropped so the caller can proceed with whatever techniques exist. Args: context (ScenarioContext): The resolved runtime inputs for this run. @@ -160,15 +160,9 @@ def resolve_technique_factories( all_factories = dict(AttackTechniqueRegistry.get_registry_singleton().get_factories_or_raise()) if extra_factories: all_factories.update(extra_factories) - missing = [ - technique.value - for technique in context.scenario_techniques - if technique.value not in all_factories - ] + missing = [technique.value for technique in context.scenario_techniques if technique.value not in all_factories] if missing: - logger.warning( - f"Selected techniques have no registered factory and will be skipped: {missing}" - ) + logger.warning(f"Selected techniques have no registered factory and will be skipped: {missing}") return { technique.value: all_factories[technique.value] for technique in context.scenario_techniques diff --git a/pyrit/score/response_handler.py b/pyrit/score/response_handler.py index 63ae53a785..da51b63e43 100644 --- a/pyrit/score/response_handler.py +++ b/pyrit/score/response_handler.py @@ -220,7 +220,7 @@ def parse( parsed category is not a string or a list of strings. InvalidJsonException: If the response is invalid JSON, is not a top-level JSON object, is missing a required key, or (when this handler is numeric) the score value is not - parsable as a float. + a finite float. """ response_json = remove_markdown_json(response_text) try: diff --git a/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py b/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py index 8b75c9174d..2d2c78ddc5 100644 --- a/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py +++ b/tests/unit/scenario/core/test_matrix_atomic_attack_builder.py @@ -15,11 +15,10 @@ * optional baseline emission from the flattened seed groups. """ +import logging from types import SimpleNamespace from unittest.mock import MagicMock, patch -import logging - import pytest from pyrit.models import AttackSeedGroup, SeedObjective @@ -410,7 +409,7 @@ def test_drops_techniques_without_factory(self): resolved = resolve_technique_factories(context=context) assert list(resolved.keys()) == ["alpha"] - def test_warns_when_selected_technique_has_no_factory(self, caplog, ): + def test_warns_when_selected_technique_has_no_factory(self, caplog: pytest.LogCaptureFixture) -> None: factories = {"alpha": _mock_factory(name="alpha")} context = _context(techniques=[_technique("alpha"), _technique("missing")]) with (