From 9a41c4347203bdc66c9027cd47332baeff51258e Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sun, 11 Oct 2026 12:42:04 -0700 Subject: [PATCH 1/2] TEST: Separate score cancellation watchdog from attack startup Wait for validation readiness before timing cleanup. Assert the validation read was cancelled and query stored scores with an explicit filter. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 226dfd07-b984-4cc9-a0f3-502d3c02d318 --- .../attack/test_scoring_expectation_transport.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/unit/executor/attack/test_scoring_expectation_transport.py b/tests/unit/executor/attack/test_scoring_expectation_transport.py index e64caa6766..e114498421 100644 --- a/tests/unit/executor/attack/test_scoring_expectation_transport.py +++ b/tests/unit/executor/attack/test_scoring_expectation_transport.py @@ -12,6 +12,7 @@ import pytest from aiosqlite import Cursor from sqlalchemy import event +from unit.async_utils import wait_for_completion_async from unit.mocks import MockPromptTarget, store_message_async from pyrit.exceptions import ( @@ -600,6 +601,7 @@ async def test_attack_error_persists_after_cancelled_score_validation_async( attack_scoring_config=AttackScoringConfig(objective_scorer=objective, auxiliary_scorers=[auxiliary]), ) write_started, validation_started, release_read = asyncio.Event(), asyncio.Event(), asyncio.Event() + validation_cancelled = asyncio.Event() original = ValueError("scoring failed") def record_write(*args: Any) -> None: @@ -615,7 +617,11 @@ async def fail_auxiliary_async(**_kwargs: Any) -> list[Score]: async def delayed_fetchall_async(cursor: Cursor) -> Iterable[Any]: if write_started.is_set() and not validation_started.is_set(): validation_started.set() - await release_read.wait() + try: + await release_read.wait() + except asyncio.CancelledError: + validation_cancelled.set() + raise return await fetchall(cursor) engine = sqlite_instance._get_async_engine() @@ -629,8 +635,12 @@ async def delayed_fetchall_async(cursor: Cursor) -> Iterable[Any]: attack.execute_async(objective="attack objective", expectation=_expectation()) ) try: + # Time cancellation cleanup separately from attack startup and target persistence. + await asyncio.wait_for(validation_started.wait(), timeout=30) with pytest.raises(RuntimeError, match="Strategy execution failed for auxiliary_scorer") as raised: - await asyncio.wait_for(task, timeout=5) + await wait_for_completion_async(future=task, timeout=5) + assert validation_cancelled.is_set() + assert not release_read.is_set() finally: release_read.set() if not task.done(): @@ -645,7 +655,7 @@ async def delayed_fetchall_async(cursor: Cursor) -> Iterable[Any]: assert stored.outcome == AttackOutcome.ERROR assert stored.error_message is not None assert "scoring failed" in stored.error_message - assert await sqlite_instance.get_scores_async() == [] + assert await sqlite_instance.get_scores_async(score_type="true_false") == [] @pytest.mark.parametrize("duplicate", [False, True], ids=["initial_node", "duplicated_node"]) @pytest.mark.parametrize("explicit_expectation", [False, True], ids=["default_objective", "scoring_objective"]) From 9ba90d8fa4748f61cdb5e454f40219668268bad2 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Mon, 12 Oct 2026 12:07:57 -0700 Subject: [PATCH 2/2] TEST: Separate analytics correctness from short query budgets Use a finite shared watchdog for analytics correctness queries while preserving dedicated deadline and cancellation coverage. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 6d66ad46-6cb7-40a5-9290-e6d67dd48ab9 --- doc/contributing/4_running_tests.md | 4 ++++ tests/unit/memory/test_attack_analytics.py | 15 ++++++++++++++- .../unit/memory/test_attack_analytics_metadata.py | 7 +------ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/doc/contributing/4_running_tests.md b/doc/contributing/4_running_tests.md index a4d3bd2685..a056340145 100644 --- a/doc/contributing/4_running_tests.md +++ b/doc/contributing/4_running_tests.md @@ -71,6 +71,10 @@ The helper's scenarios can also run directly with For a narrower run, invoke `pytest` directly. You can invoke pytest if it's in your path or via python; either `pytest` or `python -m pytest`. For the following examples, we will use `pytest`. +Analytics correctness tests use a shared, finite 120-second query budget to allow for +SQL compilation and scheduling delays on busy CI workers. Deadline and cancellation +tests use explicit controls and keep their separate timing assertions. + * To run every test under a subdirectory: ```bash diff --git a/tests/unit/memory/test_attack_analytics.py b/tests/unit/memory/test_attack_analytics.py index d8ff984804..6e584698ea 100644 --- a/tests/unit/memory/test_attack_analytics.py +++ b/tests/unit/memory/test_attack_analytics.py @@ -100,7 +100,20 @@ def make_result( def control() -> QueryControl: - return QueryControl(deadline=time.monotonic() + 10) + """Bound correctness queries without imposing a short CI performance limit.""" + return QueryControl(deadline=time.monotonic() + 120) + + +def test_correctness_query_control_retains_a_finite_deadline() -> None: + with patch.object(time, "monotonic", return_value=100): + query_control = control() + assert query_control.deadline == 220 + with patch.object(time, "monotonic", return_value=135): + query_control.check() + assert query_control.remaining == 85 + with patch.object(time, "monotonic", return_value=220): + with pytest.raises(AnalyticsTimeoutException): + query_control.check() def predicate(*, name: str, values: list[str | None], **options: Any) -> dict[str, Any]: diff --git a/tests/unit/memory/test_attack_analytics_metadata.py b/tests/unit/memory/test_attack_analytics_metadata.py index 43fc01cc0f..78306665d1 100644 --- a/tests/unit/memory/test_attack_analytics_metadata.py +++ b/tests/unit/memory/test_attack_analytics_metadata.py @@ -5,7 +5,6 @@ import json import re -import time import uuid from datetime import UTC, datetime from typing import TYPE_CHECKING, Any @@ -30,7 +29,6 @@ ScenarioResultEntry, TargetIdentifierEntry, ) -from pyrit.memory.query_control import QueryControl from pyrit.models import ( AtomicAttackIdentifier, AttackAnalyticsDimension, @@ -47,6 +45,7 @@ ConverterIdentifier, TargetIdentifier, ) +from unit.memory.test_attack_analytics import control as _control if TYPE_CHECKING: from sqlalchemy.engine import Connection @@ -80,10 +79,6 @@ def _result( ) -def _control() -> QueryControl: - return QueryControl(deadline=time.monotonic() + 30) - - def _filters(*, values: list[tuple[AttackAnalyticsDimension, str]]) -> AttackAnalyticsFilters: return AttackAnalyticsFilters( dimensions=[