Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 13 additions & 3 deletions tests/unit/executor/attack/test_scoring_expectation_transport.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
import pytest
from aiosqlite import Cursor
from sqlalchemy import event
from unit.async_utils import wait_for_completion_async
from unit.mocks import MockPromptTarget, store_message_async

from pyrit.exceptions import (
Expand Down Expand Up @@ -600,6 +601,7 @@ async def test_attack_error_persists_after_cancelled_score_validation_async(
attack_scoring_config=AttackScoringConfig(objective_scorer=objective, auxiliary_scorers=[auxiliary]),
)
write_started, validation_started, release_read = asyncio.Event(), asyncio.Event(), asyncio.Event()
validation_cancelled = asyncio.Event()
original = ValueError("scoring failed")

def record_write(*args: Any) -> None:
Expand All @@ -615,7 +617,11 @@ async def fail_auxiliary_async(**_kwargs: Any) -> list[Score]:
async def delayed_fetchall_async(cursor: Cursor) -> Iterable[Any]:
if write_started.is_set() and not validation_started.is_set():
validation_started.set()
await release_read.wait()
try:
await release_read.wait()
except asyncio.CancelledError:
validation_cancelled.set()
raise
return await fetchall(cursor)

engine = sqlite_instance._get_async_engine()
Expand All @@ -629,8 +635,12 @@ async def delayed_fetchall_async(cursor: Cursor) -> Iterable[Any]:
attack.execute_async(objective="attack objective", expectation=_expectation())
)
try:
# Time cancellation cleanup separately from attack startup and target persistence.
await asyncio.wait_for(validation_started.wait(), timeout=30)
with pytest.raises(RuntimeError, match="Strategy execution failed for auxiliary_scorer") as raised:
await asyncio.wait_for(task, timeout=5)
await wait_for_completion_async(future=task, timeout=5)
assert validation_cancelled.is_set()
assert not release_read.is_set()
finally:
release_read.set()
if not task.done():
Expand All @@ -645,7 +655,7 @@ async def delayed_fetchall_async(cursor: Cursor) -> Iterable[Any]:
assert stored.outcome == AttackOutcome.ERROR
assert stored.error_message is not None
assert "scoring failed" in stored.error_message
assert await sqlite_instance.get_scores_async() == []
assert await sqlite_instance.get_scores_async(score_type="true_false") == []

@pytest.mark.parametrize("duplicate", [False, True], ids=["initial_node", "duplicated_node"])
@pytest.mark.parametrize("explicit_expectation", [False, True], ids=["default_objective", "scoring_objective"])
Expand Down
Loading