diff --git a/pyrit/converter/text_selection_strategy.py b/pyrit/converter/text_selection_strategy.py index cdfd6abedb..85d8111de7 100644 --- a/pyrit/converter/text_selection_strategy.py +++ b/pyrit/converter/text_selection_strategy.py @@ -301,6 +301,8 @@ def __init__(self, *, proportion: float, anchor: str = "start", seed: int | None - 'middle': Select from the middle - 'random': Select from a random position seed (int | None): Random seed for reproducible random selections. Defaults to None. + Scoped to this strategy: it makes this strategy reproducible without affecting + the randomness of any other component. Raises: ValueError: If proportion is not between 0.0 and 1.0, or anchor is invalid. @@ -315,6 +317,9 @@ def __init__(self, *, proportion: float, anchor: str = "start", seed: int | None self._proportion = proportion self._anchor = anchor self._seed = seed + # Own the RNG rather than seeding the global one, so a seeded strategy + # does not make every other `random`-based component reproducible. + self._rng = random.Random(seed) def select_range(self, *, text: str) -> tuple[int, int]: """ @@ -338,9 +343,9 @@ def select_range(self, *, text: str) -> tuple[int, int]: return (start, start + selection_len) # random if self._seed is not None: - random.seed(self._seed) + self._rng.seed(self._seed) max_start = max(0, text_len - selection_len) - start = random.randint(0, max_start) if max_start > 0 else 0 + start = self._rng.randint(0, max_start) if max_start > 0 else 0 return (start, start + selection_len) @@ -479,6 +484,8 @@ def __init__(self, *, proportion: float, seed: int | None = None) -> None: Args: proportion (float): The proportion of words to select (0.0 to 1.0). seed (int | None): Random seed for reproducible selections. Defaults to None. + Scoped to this strategy: it makes this strategy reproducible without affecting + the randomness of any other component. Raises: ValueError: If proportion is not between 0.0 and 1.0. @@ -488,6 +495,9 @@ def __init__(self, *, proportion: float, seed: int | None = None) -> None: self._proportion = proportion self._seed = seed + # Own the RNG rather than seeding the global one, so a seeded strategy + # does not make every other `random`-based component reproducible. + self._rng = random.Random(seed) def select_words(self, *, words: list[str]) -> list[int]: """ @@ -503,10 +513,10 @@ def select_words(self, *, words: list[str]) -> list[int]: return [] if self._seed is not None: - random.seed(self._seed) + self._rng.seed(self._seed) num_to_select = int(len(words) * self._proportion) - return random.sample(range(len(words)), num_to_select) if num_to_select > 0 else [] + return self._rng.sample(range(len(words)), num_to_select) if num_to_select > 0 else [] class WordRegexSelectionStrategy(WordSelectionStrategy): diff --git a/pyrit/converter/zalgo_converter.py b/pyrit/converter/zalgo_converter.py index 29c9c70ee2..8468b5c11d 100644 --- a/pyrit/converter/zalgo_converter.py +++ b/pyrit/converter/zalgo_converter.py @@ -34,13 +34,22 @@ def __init__( Args: intensity (int): Number of combining marks per character (higher = more cursed). Default is 10. - seed (int | None): Optional seed for reproducible output. + seed (int | None): Optional seed for this converter's own randomness, i.e. which combining + marks are applied and how many. Seeds are component-scoped: they make the seeded + component reproducible without touching any other component's randomness. The default + word selection strategy selects every word and is deterministic, so a seed alone makes + the output reproducible. If you supply a ``word_selection_strategy`` that draws randomly + (such as ``WordProportionSelectionStrategy``), seed that strategy as well for + end-to-end reproducibility. word_selection_strategy (WordSelectionStrategy | None): Strategy for selecting which words to convert. If None, all words will be converted. """ super().__init__(word_selection_strategy=word_selection_strategy) self._intensity = self._normalize_intensity(intensity) self._seed = seed + # Own the RNG rather than seeding the global one, so a seeded converter + # does not make every other `random`-based component reproducible. + self._rng = random.Random(seed) def _build_identifier(self) -> ComponentIdentifier: """ @@ -83,12 +92,15 @@ async def convert_word_async(self, word: str) -> str: return word def glitch(char: str) -> str: - return char + "".join(random.choice(self.ZALGO_MARKS) for _ in range(random.randint(1, self._intensity))) + return char + "".join( + self._rng.choice(self.ZALGO_MARKS) for _ in range(self._rng.randint(1, self._intensity)) + ) return "".join(glitch(c) if c.isalnum() else c for c in word) def validate_input(self, prompt: str) -> None: """Validate the input prompt before conversion.""" - # Initialize the random seed before processing any words + # Reset the converter's own RNG before processing any words, so a seeded + # converter yields the same output on every call. if self._seed is not None: - random.seed(self._seed) + self._rng.seed(self._seed) diff --git a/tests/unit/converter/test_char_swap_generator_converter.py b/tests/unit/converter/test_char_swap_generator_converter.py index ad23e6154a..de96689b31 100644 --- a/tests/unit/converter/test_char_swap_generator_converter.py +++ b/tests/unit/converter/test_char_swap_generator_converter.py @@ -136,14 +136,17 @@ async def test_char_swap_converter_proportion_unchanged_with_iterations(): prompt = "Testing multiple words here today" # 50% proportion should select ~2-3 of the 5 eligible words, regardless of max_iterations - converter = CharSwapConverter( - max_iterations=10, - word_selection_strategy=WordProportionSelectionStrategy(proportion=0.5), - ) - - # Mock random.sample to select exactly 2 words (indices 0 and 2) - # This simulates the word selection strategy picking "Testing" and "words" - with patch("random.sample", return_value=[0, 2]) as mock_sample, patch("random.randint", return_value=1): + strategy = WordProportionSelectionStrategy(proportion=0.5) + converter = CharSwapConverter(max_iterations=10, word_selection_strategy=strategy) + + # Mock the strategy's own RNG to select exactly 2 words (indices 0 and 2). + # This simulates the word selection strategy picking "Testing" and "words". + # The strategy draws from a private Random instance rather than the global + # `random` module, so that a seeded strategy cannot disturb process-wide state. + with ( + patch.object(strategy._rng, "sample", return_value=[0, 2]) as mock_sample, + patch("random.randint", return_value=1), + ): result = await converter.convert_async(prompt=prompt) # Verify sample was called once (word selection happens once, not per iteration) diff --git a/tests/unit/converter/test_text_selection_strategy.py b/tests/unit/converter/test_text_selection_strategy.py index ef90e2d2d9..e0607e2cd3 100644 --- a/tests/unit/converter/test_text_selection_strategy.py +++ b/tests/unit/converter/test_text_selection_strategy.py @@ -1,6 +1,8 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import random + import pytest from pyrit.converter.text_selection_strategy import ( @@ -219,6 +221,23 @@ def test_select_range_random_anchor_with_seed(self): result2 = strategy2.select_range(text="0123456789") assert result1 == result2 # Same seed should give same result + def test_select_range_seed_does_not_disturb_global_rng(self): + """A seeded strategy must not reseed the process-wide RNG.""" + original_state = random.getstate() + try: + # Seed to a value distinct from the strategy's own seed. Without this the + # assertion can pass vacuously: a leaking strategy that reseeds to the same + # value the previous test used lands back on the captured state. + random.seed(0) + state_before = random.getstate() + + ProportionSelectionStrategy(proportion=0.3, anchor="random", seed=42).select_range(text="0123456789") + + assert random.getstate() == state_before + finally: + # Leave process-wide RNG exactly as found so test order stays irrelevant. + random.setstate(original_state) + def test_select_range_random_anchor_different_seeds(self): strategy1 = ProportionSelectionStrategy(proportion=0.3, anchor="random", seed=42) strategy2 = ProportionSelectionStrategy(proportion=0.3, anchor="random", seed=43) @@ -383,6 +402,24 @@ def test_select_words_reproducible_with_seed(self): result2 = strategy2.select_words(words=words) assert result1 == result2 + def test_select_words_seed_does_not_disturb_global_rng(self): + """A seeded strategy must not reseed the process-wide RNG.""" + original_state = random.getstate() + try: + # Seed to a value distinct from the strategy's own seed. Without this the + # assertion can pass vacuously: a leaking strategy that reseeds to the same + # value the previous test used lands back on the captured state. + random.seed(0) + state_before = random.getstate() + + strategy = WordProportionSelectionStrategy(proportion=0.3, seed=42) + strategy.select_words(words=["a", "b", "c", "d", "e", "f", "g", "h", "i", "j"]) + + assert random.getstate() == state_before + finally: + # Leave process-wide RNG exactly as found so test order stays irrelevant. + random.setstate(original_state) + def test_select_words_zero_proportion(self): strategy = WordProportionSelectionStrategy(proportion=0.0, seed=42) words = ["a", "b", "c", "d", "e"] diff --git a/tests/unit/converter/test_zalgo_converter.py b/tests/unit/converter/test_zalgo_converter.py index 4295fbe043..a25764fa0f 100644 --- a/tests/unit/converter/test_zalgo_converter.py +++ b/tests/unit/converter/test_zalgo_converter.py @@ -1,9 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. +import random + import pytest from pyrit.converter import ZalgoConverter +from pyrit.converter.text_selection_strategy import WordProportionSelectionStrategy async def test_zalgo_output_changes_text(): @@ -23,6 +26,66 @@ async def test_zalgo_reproducible_seed(): assert result1.output_text == result2.output_text +async def test_zalgo_seed_does_not_disturb_global_rng(): + """A seeded converter must not reseed the process-wide RNG.""" + original_state = random.getstate() + try: + # Seed to a value distinct from the converter's own seed. Without this the + # assertion can pass vacuously: a leaking component that reseeds to the same + # value the previous test used lands back on the captured state. + random.seed(0) + state_before = random.getstate() + + converter = ZalgoConverter(intensity=5, seed=42) + await converter.convert_async(prompt="seed test") + + assert random.getstate() == state_before + finally: + # Leave process-wide RNG exactly as found so test order stays irrelevant. + random.setstate(original_state) + + +async def test_zalgo_seed_is_repeatable_on_same_instance(): + prompt = "seed test" + converter = ZalgoConverter(intensity=5, seed=123) + first = await converter.convert_async(prompt=prompt) + second = await converter.convert_async(prompt=prompt) + assert first.output_text == second.output_text + + +async def test_zalgo_unseeded_converters_stay_independent(): + """An unseeded converter must keep producing varied output.""" + converter = ZalgoConverter(intensity=5) + outputs = {(await converter.convert_async(prompt="seed test")).output_text for _ in range(5)} + assert len(outputs) > 1 + + +async def test_zalgo_seed_is_component_scoped_when_composed_with_random_selection(): + """ + Seeds are component-scoped: seeding the converter alone does not seed a randomized + word selection strategy, but seeding both makes the composition reproducible. + """ + # Eight words at proportion 0.5 gives C(8,4)=70 possible selections, so five draws + # collapsing to one output by chance is ~4e-8 -- not a realistic flake. + prompt = "alpha bravo charlie delta echo foxtrot golf hotel" + + unseeded_selection = ZalgoConverter( + intensity=3, + seed=42, + word_selection_strategy=WordProportionSelectionStrategy(proportion=0.5), + ) + varied = {(await unseeded_selection.convert_async(prompt=prompt)).output_text for _ in range(5)} + assert len(varied) > 1 + + seeded_selection = ZalgoConverter( + intensity=3, + seed=42, + word_selection_strategy=WordProportionSelectionStrategy(proportion=0.5, seed=7), + ) + repeated = {(await seeded_selection.convert_async(prompt=prompt)).output_text for _ in range(5)} + assert len(repeated) == 1 + + async def test_zalgo_zero_intensity_returns_original(): prompt = "no chaos please" converter = ZalgoConverter(intensity=0)