diff --git a/.github/instructions/scenarios.instructions.md b/.github/instructions/scenarios.instructions.md index fa3df99322..e2673e58bc 100644 --- a/.github/instructions/scenarios.instructions.md +++ b/.github/instructions/scenarios.instructions.md @@ -169,8 +169,24 @@ Options: - Ingredients must remain complete. Use `max_per_dataset="all"` and no finite source `max_size`; cap the assembled attack groups with `max_total`, not ingredient rows. - Use `with_overrides()` instead of reconstructing a subclass or mutating its defaults. +- Clients and Python use `ScenarioDatasetRequest` through the common `dataset_overrides` + parameter. `dataset_config` still replaces the default; overrides apply on top. +- Resolve parameter/technique-dependent names in `_resolve_default_dataset_config()`, + before overrides, validation, and preparation. Do not rewrite user selections in validators. +- Declare `DATASET_OPTIONS` (or parameter-aware `get_dataset_options()`): `ANY` only for + generic named populations, `SUBSET` for approved alternatives, `FIXED` for ingredients. + Disable limits that would truncate ingredients. Compound source edits are ambiguous; + preserve child shaping/caps and allow only supported global edits. +- Estimates expose `dataset_defaults`, `effective_datasets`, and `dataset_options`. + These are response projections, not incoming edit state. Never read seeds for projections. + The catalog reads options and defaults from registry metadata, never from an estimate. +- Resume replays the saved launch request; the stored run plan fixes the selected seed groups, + so caps and fetch policies cannot change a resumed run. Dataset names are checked by the + stored scenario identity. Resume never prepares datasets. - Keep custom shaping in `_build_attack_groups()` or `_build_groups_by_dataset_async()`. Reads use `_collect_seeds_for_dataset_async()` and must not fetch. +- Build synthesized prompts from those filtered seeds, not a second unfiltered memory query. + Scenario code must not call memory seed-read methods directly; the scenario contract test enforces this. - `dataset_names`, `max_dataset_size`, `auto_fetch`, and `per_dataset()` are deprecated. `max_dataset_size` is an exact alias for `max_total`; it does not disable source caps. diff --git a/doc/code/scenarios/0_scenarios.ipynb b/doc/code/scenarios/0_scenarios.ipynb index b088db7741..7b0b110a57 100644 --- a/doc/code/scenarios/0_scenarios.ipynb +++ b/doc/code/scenarios/0_scenarios.ipynb @@ -66,10 +66,11 @@ "\n", "3. **Default Dataset**: Pass `default_dataset_config=` to `super().__init__()` to specify the datasets your scenario uses out of the box.\n", " - Returns a `DatasetAttackConfiguration` with named sources (e.g., `sources=[DatasetSource(name=\"my_dataset\")]`)\n", - " - Users can override this at runtime via `--dataset-names` in the CLI or by passing a custom `dataset_config` programmatically\n", + " - Users select sources with `--datasets name[:N|:all]`, or pass `dataset_overrides=ScenarioDatasetRequest(...)` in `set_params_from_args`. A custom `dataset_config` replaces the default; overrides apply on top.\n", " - Named sources select at most 5 attack groups per dataset by default; `max_total` caps the union. For any limit, omitted/`None`/empty/`\"default\"` uses the default; `\"all\"` removes that limit.\n", " - The deprecated `max_dataset_size` is an exact alias for `max_total`, not a way to disable source caps. To keep the old total-only selection, use `max_per_dataset=\"all\", max_total=10`; to remove both caps, set both to `\"all\"`. `None` now uses the default, not unlimited selection.\n", " - New runs prepare missing registered datasets once; reads, estimates, and resume never fetch\n", + " - Scenarios declare allowed source selections and limit controls. Estimates return resolved `dataset_defaults` and `effective_datasets` separately from requested edits. Name-only selection preserves existing source caps and fetch policies.\n", "\n", "4. **Constructor**: Use `@apply_defaults` decorator and call `super().__init__()` with scenario metadata:\n", " - `name`: Descriptive name for your scenario\n", diff --git a/doc/code/scenarios/0_scenarios.py b/doc/code/scenarios/0_scenarios.py index 3a951368a4..a9407131da 100644 --- a/doc/code/scenarios/0_scenarios.py +++ b/doc/code/scenarios/0_scenarios.py @@ -68,10 +68,11 @@ # # 3. **Default Dataset**: Pass `default_dataset_config=` to `super().__init__()` to specify the datasets your scenario uses out of the box. # - Returns a `DatasetAttackConfiguration` with named sources (e.g., `sources=[DatasetSource(name="my_dataset")]`) -# - Users can override this at runtime via `--dataset-names` in the CLI or by passing a custom `dataset_config` programmatically +# - Users select sources with `--datasets name[:N|:all]`, or pass `dataset_overrides=ScenarioDatasetRequest(...)` in `set_params_from_args`. A custom `dataset_config` replaces the default; overrides apply on top. # - Named sources select at most 5 attack groups per dataset by default; `max_total` caps the union. For any limit, omitted/`None`/empty/`"default"` uses the default; `"all"` removes that limit. # - The deprecated `max_dataset_size` is an exact alias for `max_total`, not a way to disable source caps. To keep the old total-only selection, use `max_per_dataset="all", max_total=10`; to remove both caps, set both to `"all"`. `None` now uses the default, not unlimited selection. # - New runs prepare missing registered datasets once; reads, estimates, and resume never fetch +# - Scenarios declare allowed source selections and limit controls. Estimates return resolved `dataset_defaults` and `effective_datasets` separately from requested edits. Name-only selection preserves existing source caps and fetch policies. # # 4. **Constructor**: Use `@apply_defaults` decorator and call `super().__init__()` with scenario metadata: # - `name`: Descriptive name for your scenario diff --git a/doc/code/scenarios/1_common_scenario_parameters.ipynb b/doc/code/scenarios/1_common_scenario_parameters.ipynb index e6081ccad4..5950f87f7c 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.ipynb +++ b/doc/code/scenarios/1_common_scenario_parameters.ipynb @@ -15,7 +15,7 @@ "\n", "> **Two selection axes**: *Techniques* select attack techniques (*how* attacks run — e.g., prompt\n", "> sending, role play, TAP). *Datasets* select objectives (*what* is tested — e.g., harm categories,\n", - "> compliance topics). Use `--dataset-names` on the CLI to filter by content category.\n", + "> compliance topics). Use `--datasets` on the CLI to filter by content category.\n", "\n", "> **Running scenarios from the command line?** See the [Scanner documentation](../../scanner/0_scanner.md).\n", "\n", diff --git a/doc/code/scenarios/1_common_scenario_parameters.py b/doc/code/scenarios/1_common_scenario_parameters.py index 354df1410e..0ced660ed1 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.py +++ b/doc/code/scenarios/1_common_scenario_parameters.py @@ -19,7 +19,7 @@ # # > **Two selection axes**: *Techniques* select attack techniques (*how* attacks run — e.g., prompt # > sending, role play, TAP). *Datasets* select objectives (*what* is tested — e.g., harm categories, -# > compliance topics). Use `--dataset-names` on the CLI to filter by content category. +# > compliance topics). Use `--datasets` on the CLI to filter by content category. # # > **Running scenarios from the command line?** See the [Scanner documentation](../../scanner/0_scanner.md). # diff --git a/doc/code/scenarios/3_adaptive_scenarios.ipynb b/doc/code/scenarios/3_adaptive_scenarios.ipynb index 2c9b2bc2f5..436b55b09a 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.ipynb +++ b/doc/code/scenarios/3_adaptive_scenarios.ipynb @@ -804,7 +804,7 @@ "# Use specific datasets and limit size\n", "pyrit_scan --scenario TextAdaptive --target openai_chat \\\n", " --datasets airt_hate airt_violence \\\n", - " --max-dataset-size 10\n", + " --max-total 10\n", "```" ] } diff --git a/doc/code/scenarios/3_adaptive_scenarios.py b/doc/code/scenarios/3_adaptive_scenarios.py index 247b24b786..2614eb1d17 100644 --- a/doc/code/scenarios/3_adaptive_scenarios.py +++ b/doc/code/scenarios/3_adaptive_scenarios.py @@ -301,5 +301,5 @@ def _technique_label(result) -> str: # # Use specific datasets and limit size # pyrit_scan --scenario TextAdaptive --target openai_chat \ # --datasets airt_hate airt_violence \ -# --max-dataset-size 10 +# --max-total 10 # ``` diff --git a/doc/gui/0_gui.md b/doc/gui/0_gui.md index 33e18dd0a7..61dfc0e020 100644 --- a/doc/gui/0_gui.md +++ b/doc/gui/0_gui.md @@ -505,7 +505,29 @@ Defaults are saved in this browser, separately for each signed-in account. They A missing or changed default is shown as unavailable. Select a new default or clear it; CoPyRIT does not silently substitute a different target. If browser storage is unavailable, a warning states that the choice applies only in the current session. -Scanner forms let you override either selection for one run. **Objective Target** is the target the scenario runs against. **Adversarial Target**, under **Parameters** above **Dataset override**, is the target used to generate attacks. The adversarial selector appears only for scenarios whose available techniques use the shared default; scenarios with no such use, including the explicit-target adversarial benchmark, do not show it or send an override. **Use server default** clears the per-run adversarial override. Changes to your defaults do not change existing chats or queued/running scans. Explicit adversarial targets in a scenario or technique still take priority. Scorer targets are unchanged. +Scanner forms let you override either selection for one run. **Objective Target** is the target the scenario runs against. **Adversarial Target**, under **Parameters** above **Datasets**, is the target used to generate attacks. The adversarial selector appears only for scenarios whose available techniques use the shared default; scenarios with no such use, including the explicit-target adversarial benchmark, do not show it or send an override. **Use server default** clears the per-run adversarial override. Changes to your defaults do not change existing chats or queued/running scans. Explicit adversarial targets in a scenario or technique still take priority. Scorer targets are unchanged. + +### Scenario datasets + +The **Datasets** table shows each selected name, stored seed origins (including mixed +origins), stored example count, source limit, and fetch policy. Unknown counts for +provider-only datasets are not zero. Provider availability is separate from stored origin. +**Reload dataset metadata** refreshes the page-scoped catalog without fetching seed data. +Runtime changes and launches also invalidate this catalog. + +Use **Add dataset** for allowed alternatives or named memory datasets, **Remove** to +change the selection, and **Use default sources** to restore parameter-aware defaults. +FigStep allows one of `figstep` or `figstep_pro`; ingredient and compound sources are fixed. +Unavailable limit/source controls are not submitted. Parameter and technique changes +refresh resolved defaults without discarding edits; invalid selections display an error. + +**Max per dataset**, source limits, and **Max total** accept positive integers, `all`, +or `default`/blank. Source caps apply before the total; `all` removes only that cap. +Name-only selection retains existing explicit source settings. Fetch is `never` or +`if_missing`: only a new run may prepare missing registered datasets. Preview and launch +send the same edit-only dataset object, with harm/category filters preserved. Resume uses +the saved request and the original seed selection; a different dataset selection is +rejected. Resume never prepares datasets. #### Core Adversarial Default Override diff --git a/doc/scanner/1_pyrit_scan.ipynb b/doc/scanner/1_pyrit_scan.ipynb index ee28f8b9e7..670f985d40 100644 --- a/doc/scanner/1_pyrit_scan.ipynb +++ b/doc/scanner/1_pyrit_scan.ipynb @@ -1194,9 +1194,17 @@ "on demand, so a full dataset preload is not required:\n", "\n", "```shell\n", - "pyrit_scan run garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1\n", + "pyrit_scan run garak.figstep --target openai_chat --datasets figstep_pro --max-total 1 --dataset-fetch if_missing\n", "```\n", "\n", + "Scanner and shell share `--datasets name[:N|:all]`, `--max-per-dataset`, `--max-total`,\n", + "and `--dataset-fetch never|if_missing`. Source limits apply before the combined total.\n", + "Omitted, empty, or `default` keeps the scenario setting; `all` removes only its own cap.\n", + "`list-scenarios` displays allowed sources and defaults. Ingredient/compound scenarios\n", + "restrict controls to preserve their shaping. `--dataset-filters` works with either form.\n", + "Deprecated `--dataset-names` and `--max-dataset-size` cannot be mixed with the new flags.\n", + "Discovery and estimates never fetch; only a new run prepares absent registered datasets.\n", + "\n", "Custom initialization scripts are loaded by the backend at startup: list them in the\n", "`initialization_scripts` section of the config the server loads (paths are relative to your\n", "working directory, but full paths avoid confusion). Once the server is running with that\n", diff --git a/doc/scanner/1_pyrit_scan.py b/doc/scanner/1_pyrit_scan.py index 0aa18722aa..1e5dce2700 100644 --- a/doc/scanner/1_pyrit_scan.py +++ b/doc/scanner/1_pyrit_scan.py @@ -125,9 +125,17 @@ # on demand, so a full dataset preload is not required: # # ```shell -# pyrit_scan run garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1 +# pyrit_scan run garak.figstep --target openai_chat --datasets figstep_pro --max-total 1 --dataset-fetch if_missing # ``` # +# Scanner and shell share `--datasets name[:N|:all]`, `--max-per-dataset`, `--max-total`, +# and `--dataset-fetch never|if_missing`. Source limits apply before the combined total. +# Omitted, empty, or `default` keeps the scenario setting; `all` removes only its own cap. +# `list-scenarios` displays allowed sources and defaults. Ingredient/compound scenarios +# restrict controls to preserve their shaping. `--dataset-filters` works with either form. +# Deprecated `--dataset-names` and `--max-dataset-size` cannot be mixed with the new flags. +# Discovery and estimates never fetch; only a new run prepares absent registered datasets. +# # Custom initialization scripts are loaded by the backend at startup: list them in the # `initialization_scripts` section of the config the server loads (paths are relative to your # working directory, but full paths avoid confusion). Once the server is running with that diff --git a/doc/scanner/adaptive.ipynb b/doc/scanner/adaptive.ipynb index a4bc5849a4..1f7c72fd64 100644 --- a/doc/scanner/adaptive.ipynb +++ b/doc/scanner/adaptive.ipynb @@ -26,8 +26,8 @@ "pyrit_scan run adaptive.text_adaptive \\\n", " --initializers target \\\n", " --target openai_chat \\\n", - " --dataset-names airt_hate \\\n", - " --max-dataset-size 2 \\\n", + " --datasets airt_hate \\\n", + " --max-total 2 \\\n", " --max-attempts-per-objective 2\n", "```" ] diff --git a/doc/scanner/adaptive.py b/doc/scanner/adaptive.py index b229e0e55d..0eda326f0b 100644 --- a/doc/scanner/adaptive.py +++ b/doc/scanner/adaptive.py @@ -25,8 +25,8 @@ # pyrit_scan run adaptive.text_adaptive \ # --initializers target \ # --target openai_chat \ -# --dataset-names airt_hate \ -# --max-dataset-size 2 \ +# --datasets airt_hate \ +# --max-total 2 \ # --max-attempts-per-objective 2 # ``` diff --git a/doc/scanner/airt.ipynb b/doc/scanner/airt.ipynb index 09cf12e75e..04866a9005 100644 --- a/doc/scanner/airt.ipynb +++ b/doc/scanner/airt.ipynb @@ -102,8 +102,8 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --techniques role_play_movie_script \\\n", - " --dataset-names airt_hate \\\n", - " --max-dataset-size 1\n", + " --datasets airt_hate \\\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play_movie_script, many_shot, tap" @@ -255,8 +255,8 @@ "\n", "Each sub-harm escalates a simulated multi-turn conversation toward the objective, then layers the\n", "selected converter techniques on top (natural-language rewrites that preserve emotional framing;\n", - "obfuscation converters like base64/morse are excluded). Datasets are bound to the sub-harms, so\n", - "`--dataset-names` is ignored (`--max-dataset-size` still applies).\n", + "obfuscation converters like base64/morse are excluded). `--datasets` can select a subset of the\n", + "selected sub-harms' datasets; `--max-total` must cover each selected sub-harm.\n", "\n", "**Available techniques:** ALL, DEFAULT, tone, language, persuasion, deterministic, crescendo" ] @@ -413,7 +413,7 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --techniques role_play_movie_script \\\n", - " --max-dataset-size 1\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** Use `pyrit_scan run airt.cyber --list-scenario-parameters` to inspect\n", @@ -570,8 +570,8 @@ "pyrit_scan run airt.jailbreak \\\n", " --initializers target \\\n", " --target openai_chat \\\n", - " --dataset-names harmbench \\\n", - " --max-dataset-size 1\n", + " --datasets harmbench \\\n", + " --max-total 1\n", "```\n", "\n", "**Available technique selectors:** ALL, DEFAULT, and SINGLE_TURN currently select both\n", @@ -730,8 +730,8 @@ "pyrit_scan airt.multilingual \\\n", " --initializers target \\\n", " --target openai_chat \\\n", - " --dataset-names harmbench \\\n", - " --max-dataset-size 1\n", + " --datasets harmbench \\\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** `prompt_sending` is the default. Every registry technique (`role_play_*`,\n", @@ -870,7 +870,7 @@ "plagiarism detection.\n", "\n", "```bash\n", - "pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1\n", + "pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-total 1\n", "```\n", "\n", "**Available techniques:** ALL, SINGLE_TURN, MULTI_TURN, IP, SENSITIVE_DATA, FirstLetter, Image, RolePlay, Crescendo\n", @@ -1049,7 +1049,7 @@ " --initializers target \\\n", " --target openai_chat \\\n", " --techniques context_compliance \\\n", - " --max-dataset-size 1\n", + " --max-total 1\n", "```\n", "\n", "**Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay,\n", diff --git a/doc/scanner/airt.py b/doc/scanner/airt.py index cb804f14e7..a7fba16ebc 100644 --- a/doc/scanner/airt.py +++ b/doc/scanner/airt.py @@ -43,8 +43,8 @@ # --initializers target \ # --target openai_chat \ # --techniques role_play_movie_script \ -# --dataset-names airt_hate \ -# --max-dataset-size 1 +# --datasets airt_hate \ +# --max-total 1 # ``` # # **Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, role_play_movie_script, many_shot, tap @@ -84,8 +84,8 @@ # # Each sub-harm escalates a simulated multi-turn conversation toward the objective, then layers the # selected converter techniques on top (natural-language rewrites that preserve emotional framing; -# obfuscation converters like base64/morse are excluded). Datasets are bound to the sub-harms, so -# `--dataset-names` is ignored (`--max-dataset-size` still applies). +# obfuscation converters like base64/morse are excluded). `--datasets` can select a subset of the +# selected sub-harms' datasets; `--max-total` must cover each selected sub-harm. # # **Available techniques:** ALL, DEFAULT, tone, language, persuasion, deterministic, crescendo @@ -124,7 +124,7 @@ # --initializers target \ # --target openai_chat \ # --techniques role_play_movie_script \ -# --max-dataset-size 1 +# --max-total 1 # ``` # # **Available techniques:** Use `pyrit_scan run airt.cyber --list-scenario-parameters` to inspect @@ -169,8 +169,8 @@ # pyrit_scan run airt.jailbreak \ # --initializers target \ # --target openai_chat \ -# --dataset-names harmbench \ -# --max-dataset-size 1 +# --datasets harmbench \ +# --max-total 1 # ``` # # **Available technique selectors:** ALL, DEFAULT, and SINGLE_TURN currently select both @@ -213,8 +213,8 @@ # pyrit_scan airt.multilingual \ # --initializers target \ # --target openai_chat \ -# --dataset-names harmbench \ -# --max-dataset-size 1 +# --datasets harmbench \ +# --max-total 1 # ``` # # **Available techniques:** `prompt_sending` is the default. Every registry technique (`role_play_*`, @@ -254,7 +254,7 @@ # plagiarism detection. # # ```bash -# pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-dataset-size 1 +# pyrit_scan run airt.leakage --target openai_chat --techniques first_letter --max-total 1 # ``` # # **Available techniques:** ALL, SINGLE_TURN, MULTI_TURN, IP, SENSITIVE_DATA, FirstLetter, Image, RolePlay, Crescendo @@ -308,7 +308,7 @@ # --initializers target \ # --target openai_chat \ # --techniques context_compliance \ -# --max-dataset-size 1 +# --max-total 1 # ``` # # **Available techniques:** ALL, DEFAULT, SINGLE_TURN, MULTI_TURN, ContextCompliance, RolePlay, diff --git a/doc/scanner/benchmark.ipynb b/doc/scanner/benchmark.ipynb index 6abd678fe9..2ed6d84aeb 100644 --- a/doc/scanner/benchmark.ipynb +++ b/doc/scanner/benchmark.ipynb @@ -35,7 +35,7 @@ " --target openai_chat \\\n", " --adversarial-targets adversarial_chat \\\n", " --techniques role_play_video_game \\\n", - " --max-dataset-size 1\n", + " --max-total 1\n", "```\n", "\n", "Pass multiple `--adversarial-targets` values to compare across models in a single run.\n", diff --git a/doc/scanner/benchmark.py b/doc/scanner/benchmark.py index 8adddffbc1..92588c50db 100644 --- a/doc/scanner/benchmark.py +++ b/doc/scanner/benchmark.py @@ -34,7 +34,7 @@ # --target openai_chat \ # --adversarial-targets adversarial_chat \ # --techniques role_play_video_game \ -# --max-dataset-size 1 +# --max-total 1 # ``` # # Pass multiple `--adversarial-targets` values to compare across models in a single run. diff --git a/doc/scanner/foundry.ipynb b/doc/scanner/foundry.ipynb index 78a5dbeaab..dca0cb3342 100644 --- a/doc/scanner/foundry.ipynb +++ b/doc/scanner/foundry.ipynb @@ -73,7 +73,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1\n", + "pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-total 1\n", "```\n", "\n", "**Available techniques by difficulty:**\n", diff --git a/doc/scanner/foundry.py b/doc/scanner/foundry.py index bbcc9b8bb6..e73747b8ca 100644 --- a/doc/scanner/foundry.py +++ b/doc/scanner/foundry.py @@ -40,7 +40,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-dataset-size 1 +# pyrit_scan run foundry.red_team_agent --target openai_chat --techniques base64 --max-total 1 # ``` # # **Available techniques by difficulty:** diff --git a/doc/scanner/garak.ipynb b/doc/scanner/garak.ipynb index 0425b4e870..da4ca4811d 100644 --- a/doc/scanner/garak.ipynb +++ b/doc/scanner/garak.ipynb @@ -37,7 +37,7 @@ "PromptInject tests whether a target follows injected instructions hidden in benign tasks.\n", "`--techniques` selects the injection wording, and `--goal-texts` selects the requested output.\n", "The default run uses all five techniques, the three Garak goal texts, and up to 60 attempts.\n", - "`--max-dataset-size` must be at least the number of selected goal texts.\n", + "`--max-total` must be at least the number of selected goal texts.\n", "Sampling selects at least one context for every goal, then fills the remaining budget\n", "without replacement. All techniques use the same sample. Injection wording is applied\n", "through request converters, before any caller-supplied converters.\n", @@ -49,7 +49,7 @@ "pyrit_scan run garak.prompt_inject --target openai_chat \\\n", " --techniques ignore_print \\\n", " --goal-texts \"Custom trigger text\" \\\n", - " --max-dataset-size 16\n", + " --max-total 16\n", "```\n", "\n", "**Techniques:** `ignore_print`, `ignore_say`, `ignore_print_upper`, `nevermind`, and\n", @@ -299,7 +299,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1\n", + "pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-total 1\n", "```\n", "\n", "**Available techniques** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex,\n", @@ -457,8 +457,8 @@ "**CLI examples:**\n", "\n", "```bash\n", - "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep --max-dataset-size 1\n", - "pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1\n", + "pyrit_scan garak.figstep --target openai_chat --datasets figstep --max-total 1\n", + "pyrit_scan garak.figstep --target openai_chat --datasets figstep_pro --max-total 1\n", "```\n", "\n", "> **Note:** The objective target must natively support multi-piece user messages and accept text\n", @@ -620,7 +620,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1\n", + "pyrit_scan run garak.web_injection --target openai_chat --techniques xss\n", "```\n", "\n", "**Available techniques** (8 probes): MarkdownImageExfil, ColabAIDataLeakage,\n", @@ -630,7 +630,7 @@ "**Aggregate techniques:** `ALL` (all 8), `DEFAULT` (excludes the two combinatorial extended\n", "probes), `EXFIL` (the 6 markdown-exfil probes), and `XSS` (TaskXSS + MarkdownXSS).\n", "\n", - "**Dataset requirements:** `--dataset-names` replaces the default datasets, so it must include\n", + "**Dataset requirements:** `--datasets` replaces the default datasets, so it must include\n", "every dataset the selected techniques read. Otherwise the scenario stops before running and names\n", "each missing dataset. Datasets already loaded into memory never fill the gap.\n", "\n", @@ -645,7 +645,7 @@ "| StringAssemblyDataExfil | None (built-in seeds) |\n", "\n", "```bash\n", - "pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --dataset-names garak_xss_normal_instructions garak_web_html_js\n", + "pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --datasets garak_xss_normal_instructions garak_web_html_js\n", "```" ] }, @@ -986,7 +986,7 @@ "pyrit_scan run garak.api_key --target openai_chat\n", "\n", "# Run only GetKey with a smaller sample.\n", - "pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-dataset-size 2\n", + "pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-total 2\n", "```\n", "\n", "**Available techniques:** `GetKey` and `CompleteKey`. `DEFAULT` and `ALL` both select the two\n", @@ -1161,7 +1161,7 @@ "\n", "```bash\n", "pyrit_scan run garak.latent_injection --target openai_chat --techniques bare \\\n", - " --families whois --max-dataset-size 1\n", + " --families whois --max-total 1\n", "```\n", "\n", "**Available techniques** (14 separator styles): Bare, Newline, Blockquote, BlockquoteInline,\n", @@ -1353,7 +1353,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1\n", + "pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-total 1\n", "```\n", "\n", "**Available techniques** (2 probes): `PolicyPuppetry` (wraps the objective in the Dr House\n", @@ -1845,13 +1845,13 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-dataset-size 2\n", + "pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-total 2\n", "```\n", "\n", "> **Note:** The objective target must accept `audio_path` input (i.e. be multimodal). The example\n", "> below uses the registered Azure OpenAI Realtime target; non-audio targets such as the default\n", "> `openai_chat` will error when the audio request is sent. The full dataset holds ~350\n", - "> clips, so a default run samples a small subset to finish quickly — raise `--max-dataset-size`\n", + "> clips, so a default run samples a small subset to finish quickly — raise `--max-total`\n", "> for broader coverage." ] }, @@ -2009,7 +2009,7 @@ "**CLI example:**\n", "\n", "```bash\n", - "pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-dataset-size 2\n", + "pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-total 2\n", "```\n", "\n", "**Available techniques:** `Repeat`, `DEFAULT`, and `ALL` all select the same probe.\n", diff --git a/doc/scanner/garak.py b/doc/scanner/garak.py index 38673b15cd..5276a808ae 100644 --- a/doc/scanner/garak.py +++ b/doc/scanner/garak.py @@ -36,7 +36,7 @@ # PromptInject tests whether a target follows injected instructions hidden in benign tasks. # `--techniques` selects the injection wording, and `--goal-texts` selects the requested output. # The default run uses all five techniques, the three Garak goal texts, and up to 60 attempts. -# `--max-dataset-size` must be at least the number of selected goal texts. +# `--max-total` must be at least the number of selected goal texts. # Sampling selects at least one context for every goal, then fills the remaining budget # without replacement. All techniques use the same sample. Injection wording is applied # through request converters, before any caller-supplied converters. @@ -48,7 +48,7 @@ # pyrit_scan run garak.prompt_inject --target openai_chat \ # --techniques ignore_print \ # --goal-texts "Custom trigger text" \ -# --max-dataset-size 16 +# --max-total 16 # ``` # # **Techniques:** `ignore_print`, `ignore_say`, `ignore_print_upper`, `nevermind`, and @@ -132,7 +132,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-dataset-size 1 +# pyrit_scan run garak.encoding --target openai_chat --techniques base64 --max-total 1 # ``` # # **Available techniques** (17 encodings): Base64, Base2048, Base16, Base32, ASCII85, Hex, @@ -180,8 +180,8 @@ # **CLI examples:** # # ```bash -# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep --max-dataset-size 1 -# pyrit_scan garak.figstep --target openai_chat --dataset-names figstep_pro --max-dataset-size 1 +# pyrit_scan garak.figstep --target openai_chat --datasets figstep --max-total 1 +# pyrit_scan garak.figstep --target openai_chat --datasets figstep_pro --max-total 1 # ``` # # > **Note:** The objective target must natively support multi-piece user messages and accept text @@ -221,7 +221,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.web_injection --target openai_chat --techniques xss --max-dataset-size 1 +# pyrit_scan run garak.web_injection --target openai_chat --techniques xss # ``` # # **Available techniques** (8 probes): MarkdownImageExfil, ColabAIDataLeakage, @@ -231,7 +231,7 @@ # **Aggregate techniques:** `ALL` (all 8), `DEFAULT` (excludes the two combinatorial extended # probes), `EXFIL` (the 6 markdown-exfil probes), and `XSS` (TaskXSS + MarkdownXSS). # -# **Dataset requirements:** `--dataset-names` replaces the default datasets, so it must include +# **Dataset requirements:** `--datasets` replaces the default datasets, so it must include # every dataset the selected techniques read. Otherwise the scenario stops before running and names # each missing dataset. Datasets already loaded into memory never fill the gap. # @@ -246,7 +246,7 @@ # | StringAssemblyDataExfil | None (built-in seeds) | # # ```bash -# pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --dataset-names garak_xss_normal_instructions garak_web_html_js +# pyrit_scan run garak.web_injection --target openai_chat --techniques task_xss --datasets garak_xss_normal_instructions garak_web_html_js # ``` # %% @@ -333,7 +333,7 @@ # pyrit_scan run garak.api_key --target openai_chat # # # Run only GetKey with a smaller sample. -# pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-dataset-size 2 +# pyrit_scan run garak.api_key --target openai_chat --techniques get_key --max-total 2 # ``` # # **Available techniques:** `GetKey` and `CompleteKey`. `DEFAULT` and `ALL` both select the two @@ -381,7 +381,7 @@ # # ```bash # pyrit_scan run garak.latent_injection --target openai_chat --techniques bare \ -# --families whois --max-dataset-size 1 +# --families whois --max-total 1 # ``` # # **Available techniques** (14 separator styles): Bare, Newline, Blockquote, BlockquoteInline, @@ -447,7 +447,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-dataset-size 1 +# pyrit_scan run garak.doctor --target openai_chat --techniques policy_puppetry --max-total 1 # ``` # # **Available techniques** (2 probes): `PolicyPuppetry` (wraps the objective in the Dr House @@ -575,13 +575,13 @@ # **CLI example:** # # ```bash -# pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-dataset-size 2 +# pyrit_scan garak.audio_achilles_heel --target azure_openai_realtime --max-total 2 # ``` # # > **Note:** The objective target must accept `audio_path` input (i.e. be multimodal). The example # > below uses the registered Azure OpenAI Realtime target; non-audio targets such as the default # > `openai_chat` will error when the audio request is sent. The full dataset holds ~350 -# > clips, so a default run samples a small subset to finish quickly — raise `--max-dataset-size` +# > clips, so a default run samples a small subset to finish quickly — raise `--max-total` # > for broader coverage. # %% @@ -620,7 +620,7 @@ # **CLI example:** # # ```bash -# pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-dataset-size 2 +# pyrit_scan run garak.divergence --target openai_chat --techniques repeat --max-total 2 # ``` # # **Available techniques:** `Repeat`, `DEFAULT`, and `ALL` all select the same probe. diff --git a/frontend/e2e/numeric-controls.spec.ts b/frontend/e2e/numeric-controls.spec.ts index a9b7cc8705..1b2d277d1a 100644 --- a/frontend/e2e/numeric-controls.spec.ts +++ b/frontend/e2e/numeric-controls.spec.ts @@ -1,11 +1,26 @@ import { expect, test, type Locator, type Page } from './_fixtures' import { mockVersion } from './_compatibility' -import type { RegisteredScenario } from '../src/types' +import type { RegisteredScenario, ScenarioDatasetRequest, ScenarioDatasetSettings } from '../src/types' import { READY_RUNTIME } from './_runtime' import { makeTarget } from './_targets' +const DATASET_DEFAULTS: ScenarioDatasetSettings = { + sources: [{ name: 'harmbench', max_size: 5, fetch: 'if_missing', configured_provider: false }], + max_per_dataset: 5, max_total: 5, fetch: 'if_missing', filters: {}, +} +const DATASET_OPTIONS = { + mode: 'any' as const, allowed_names: [], per_dataset_limit: true, + total_limit: true, supports_overrides: true, source_settings: true, filters: true, +} +const DATASETS = { + items: [ + { name: 'harmbench', selection_key: 'dataset:named:harmbench', loaded: false, provider_available: true, origins: [], logical_examples: null }, + { name: 'custom_objectives', selection_key: 'dataset:named:custom_objectives', loaded: true, provider_available: false, origins: ['user'], logical_examples: 10 }, + ], +} + const SCENARIO: RegisteredScenario = { scenario_name: 'numeric-controls', scenario_type: 'NumericControlsScenario', @@ -18,7 +33,9 @@ const SCENARIO: RegisteredScenario = { default_technique: 'prompt_sending', default_techniques: ['prompt_sending'], technique_summaries: [{ name: 'prompt_sending', description: null, tags: [] }], - default_datasets: [], + default_datasets: ['harmbench'], + dataset_defaults: DATASET_DEFAULTS, + dataset_options: DATASET_OPTIONS, baseline_policy: 'disabled', include_baseline_by_default: false, supported_parameters: [ @@ -26,12 +43,13 @@ const SCENARIO: RegisteredScenario = { { name: 'temperature', type_name: 'float', required: false, default: '0.5', choices: null, is_list: false }, ], supported_parameter_names: [ - 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'objective_target', 'scenario_techniques', 'dataset_config', 'dataset_overrides', 'memory_labels', 'max_concurrency', 'max_retries', 'include_baseline', 'iterations', 'temperature', ], default_run_size: { dataset_size: { kind: 'indeterminate', reason: 'configuration_unavailable', detail: 'Population configuration is not available.' }, - dataset_limit: { state: 'scenario_default' }, + dataset_limit: { state: 'value', value: 5 }, + dataset_defaults: DATASET_DEFAULTS, effective_datasets: DATASET_DEFAULTS, dataset_options: DATASET_OPTIONS, estimated_attack_count: null, components: [], datasets: [], @@ -74,9 +92,29 @@ async function mockNumericControlApis(page: Page): Promise { ], }, '/api/labels': { source: 'attacks', labels: {} }, + '/api/datasets': DATASETS, [`/api/scenarios/catalog/${SCENARIO.scenario_name}`]: SCENARIO, [`/api/scenarios/catalog/${SCENARIO.scenario_name}/estimate`]: SCENARIO.default_run_size, } + if (path.endsWith('/estimate')) { + const edits: ScenarioDatasetRequest = route.request().postDataJSON()?.datasets ?? {} + const limit = (value: ScenarioDatasetRequest['max_total'], fallback: number | 'all'): number | 'all' => + typeof value === 'number' || value === 'all' ? value : fallback + const effective: ScenarioDatasetSettings = { + ...DATASET_DEFAULTS, + max_per_dataset: limit(edits.max_per_dataset, DATASET_DEFAULTS.max_per_dataset), + max_total: limit(edits.max_total, DATASET_DEFAULTS.max_total), + fetch: edits.fetch === 'never' || edits.fetch === 'if_missing' ? edits.fetch : DATASET_DEFAULTS.fetch, + sources: (edits.sources ?? DATASET_DEFAULTS.sources).map((source) => ({ + name: source.name, + max_size: limit(source.max_size, DATASET_DEFAULTS.max_per_dataset), + fetch: source.fetch === 'never' || source.fetch === 'if_missing' ? source.fetch : DATASET_DEFAULTS.fetch, + configured_provider: false, + })), + } + await route.fulfill({ json: { ...SCENARIO.default_run_size, effective_datasets: effective } }) + return + } if (!(path in responses)) { await route.abort() throw new Error(`Unexpected API request: ${route.request().method()} ${path}`) @@ -168,21 +206,40 @@ test('scenario steppers commit typed values before stepping and clamp at existin await expect(page.getByRole('spinbutton', { name: 'Max concurrency', exact: true })).toHaveValue('10') }) -test('native scenario dataset and shared dynamic parameter controls step once', async ({ page }) => { +test('native shared dynamic parameter controls step once', async ({ page }) => { await page.goto(`/scanner/${SCENARIO.scenario_name}`) - const datasetSize = page.getByRole('spinbutton', { name: 'Max dataset size', exact: true }) - await expectNativeSteps(datasetSize, 2) await expectNativeSteps(page.getByRole('spinbutton', { name: 'iterations', exact: true }), 2) // The existing dynamic float input has native step=1 with a fractional base. await expectNativeSteps(page.getByRole('spinbutton', { name: 'temperature', exact: true }), 0.5) - await datasetSize.fill('1') - await clickNativeSpinner(datasetSize, 'down') - await expect(datasetSize).toHaveValue('1') - await datasetSize.press('ArrowDown') - await expect(datasetSize).toHaveValue('1') - await datasetSize.fill('') - await datasetSize.press('Tab') - await expect(datasetSize).toHaveValue('') +}) + +test('dataset limits support default and all and keep invalid text blocking after source reset', async ({ page }, testInfo) => { + await page.goto(`/scanner/${SCENARIO.scenario_name}`) + await expect(page.getByRole('table', { name: 'Selected datasets' })).toBeVisible() + await page.getByRole('combobox', { name: 'Objective Target' }).selectOption('mock-target') + const total = page.getByRole('textbox', { name: 'Max total', exact: true }) + await expect(total).toHaveValue('') + await page.getByRole('region', { name: 'Dataset configuration' }).screenshot({ path: testInfo.outputPath('datasets-default.png') }) + await total.fill('all') + await page.getByRole('combobox', { name: 'Add dataset' }).fill('custom_objectives') + await page.getByRole('button', { name: 'Add source', exact: true }).click() + await page.getByRole('textbox', { name: 'custom_objectives limit' }).fill('8') + await page.getByRole('combobox', { name: 'custom_objectives fetch' }).selectOption('never') + await total.fill('12') + await expect(page.getByText('Default/effective: 12. Blank or default retains the scenario setting.')).toBeVisible() + await expect(page.getByText('Default/effective: 8. Blank or default retains the scenario setting.')).toBeVisible() + await page.getByRole('region', { name: 'Dataset configuration' }).screenshot({ path: testInfo.outputPath('datasets-edited.png') }) + await total.fill('0') + await expect(page.getByRole('alert')).toContainText('Max total must be a positive integer') + await page.getByRole('button', { name: 'Use default sources' }).click() + await expect(total).toHaveValue('0') + await expect(page.getByRole('alert')).toContainText('Max total must be a positive integer') + await expect(page.getByRole('button', { name: 'Launch scan', exact: true })).toBeDisabled() + await page.getByRole('region', { name: 'Dataset configuration' }).screenshot({ path: testInfo.outputPath('datasets-invalid.png') }) + await total.fill('default') + await expect(page.getByRole('alert')).not.toBeVisible() + await total.fill('') + await expect(total).toHaveValue('') }) test('native Azure ML advanced controls retain steps and reset to default guidance', async ({ page }) => { diff --git a/frontend/e2e/onboarding-tour.spec.ts b/frontend/e2e/onboarding-tour.spec.ts index dae177aaa3..534e522ea0 100644 --- a/frontend/e2e/onboarding-tour.spec.ts +++ b/frontend/e2e/onboarding-tour.spec.ts @@ -39,6 +39,9 @@ const TOUR_SCENARIO: RegisteredScenario = { /** Serves a one-scenario catalog so the detail route is reachable by click. */ async function mockScenarioCatalog(page: Page): Promise { + await page.route(/\/api\/datasets(?:\?|$)/, async (route) => { + await route.fulfill({ json: { items: [] } }); + }); await page.route(/\/api\/scenarios\/catalog(?:\?.*)?$/, async (route) => { await route.fulfill({ json: { diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts index 0cfc087e49..22d53393db 100644 --- a/frontend/e2e/scenario-history.spec.ts +++ b/frontend/e2e/scenario-history.spec.ts @@ -264,6 +264,10 @@ async function mockScenarioAPIs(page: Page): Promise { }); }); + await page.route(/\/api\/datasets(?:\?|$)/, async (route) => { + await route.fulfill({ json: { items: [] } }); + }); + await page.route(/\/api\/runtime(?:\?|$)/, async (route) => { await route.fulfill({ status: 200, diff --git a/frontend/src/components/Scenarios/ScenarioDatasetTable.styles.ts b/frontend/src/components/Scenarios/ScenarioDatasetTable.styles.ts new file mode 100644 index 0000000000..e40aea247c --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioDatasetTable.styles.ts @@ -0,0 +1,13 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTarget } from '@/styles/touchTargets' + +export const useScenarioDatasetTableStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalM }, + controls: { display: 'flex', flexWrap: 'wrap', gap: tokens.spacingHorizontalM }, + scroll: { overflowX: 'auto' }, + table: { width: '100%', borderCollapse: 'collapse' }, + cell: { padding: tokens.spacingHorizontalS, textAlign: 'left' }, + input: { maxWidth: '12em' }, + button: { ...mobileTouchTarget }, +}) diff --git a/frontend/src/components/Scenarios/ScenarioDatasetTable.test.tsx b/frontend/src/components/Scenarios/ScenarioDatasetTable.test.tsx new file mode 100644 index 0000000000..3de4f9480d --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioDatasetTable.test.tsx @@ -0,0 +1,174 @@ +import { useState } from 'react' + +import { render, screen, within } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { FluentProvider, webLightTheme } from '@fluentui/react-components' + +import type { ScenarioDatasetOptions, ScenarioDatasetRequest, ScenarioDatasetSettings } from '@/types' + +import ScenarioDatasetTable from './ScenarioDatasetTable' + +const DEFAULTS: ScenarioDatasetSettings = { + sources: [{ name: 'stored', max_size: 2, fetch: 'never', configured_provider: false }], + max_per_dataset: 5, max_total: 'all', fetch: 'if_missing', filters: {}, +} +const OPTIONS: ScenarioDatasetOptions = { + mode: 'any', allowed_names: [], per_dataset_limit: true, total_limit: true, supports_overrides: true, filters: true, +} + +interface HarnessProps { + options?: ScenarioDatasetOptions + defaults?: ScenarioDatasetSettings + onChange: jest.Mock +} + +function Harness({ options = OPTIONS, defaults = DEFAULTS, onChange }: HarnessProps) { + const [request, setRequest] = useState({}) + const [validation, setValidation] = useState(null) + return { setRequest(next); onChange(next) }} + /> +} + +describe('ScenarioDatasetTable', () => { + beforeEach(() => jest.clearAllMocks()) + + it('shows mixed origins, stored counts, and unknown provider-only counts without sending defaults', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + expect(screen.getByText('local, unknown')).toBeInTheDocument() + expect(screen.getByText('8')).toBeInTheDocument() + expect(onChange).not.toHaveBeenCalled() + await user.type(screen.getByRole('combobox', { name: 'Add dataset' }), 'remote') + await user.click(screen.getByRole('button', { name: 'Add source' })) + const row = screen.getByRole('rowheader', { name: 'remote' }).closest('tr') + expect(row).not.toBeNull() + if (row) { + expect(within(row).getByText('Provider available')).toBeInTheDocument() + expect(within(row).getAllByText('Unknown')).toHaveLength(2) + } + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ sources: [{ name: 'stored' }, { name: 'remote' }] }) + }) + + it('edits source caps and policies independently of global limits with keyboard-accessible controls', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await user.type(screen.getByRole('textbox', { name: 'stored limit' }), 'all') + await user.selectOptions(screen.getByRole('combobox', { name: 'stored fetch' }), 'if_missing') + await user.type(screen.getByRole('textbox', { name: 'Max total' }), '12') + await user.selectOptions(screen.getByRole('combobox', { name: 'Default dataset fetch' }), 'never') + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ + sources: [{ name: 'stored', max_size: 'all', fetch: 'if_missing' }], max_total: 12, fetch: 'never', + }) + await user.clear(screen.getByRole('textbox', { name: 'Max total' })) + expect(onChange.mock.calls.at(-1)?.[0]).not.toHaveProperty('max_total') + screen.getByRole('textbox', { name: 'Max per dataset' }).focus() + await user.keyboard('3') + expect(onChange.mock.calls.at(-1)?.[0]).toHaveProperty('max_per_dataset', 3) + }) + + it('does not send fixed or unsupported inputs and supports one-source replacement', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + const view = render() + expect(screen.queryByRole('button', { name: 'Add source' })).not.toBeInTheDocument() + expect(screen.queryByRole('textbox', { name: 'stored limit' })).not.toBeInTheDocument() + expect(screen.queryByRole('textbox', { name: 'Max total' })).not.toBeInTheDocument() + expect(onChange).not.toHaveBeenCalled() + view.rerender() + await user.click(screen.getByRole('combobox', { name: 'Add dataset' })) + await user.click(screen.getByRole('option', { name: 'remote' })) + await user.click(screen.getByRole('button', { name: 'Add source' })) + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({ sources: [{ name: 'remote' }] }) + expect(screen.queryByRole('rowheader', { name: 'stored' })).not.toBeInTheDocument() + }) + + it('retains edits and displays invalid selections when parameter-dependent options change', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + const view = render() + await user.type(screen.getByRole('textbox', { name: 'stored limit' }), '1') + view.rerender() + expect(screen.getByRole('alert')).toHaveTextContent('Invalid selection for current parameters: stored') + expect(screen.getByRole('textbox', { name: 'stored limit' })).toHaveValue('1') + await user.click(screen.getByRole('button', { name: 'Use default sources' })) + expect(screen.queryByRole('rowheader', { name: 'stored' })).not.toBeInTheDocument() + expect(onChange.mock.calls.at(-1)?.[0]).toEqual({}) + }) + it.each([ + ['Max total', 'max_total', '3'], ['Max total', 'max_total', 'all'], + ['Max per dataset', 'max_per_dataset', '3'], ['Max per dataset', 'max_per_dataset', 'all'], + ])('keeps invalid %s blocking launch after source reset (%s=%s)', async (label, key, initial) => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + const input = screen.getByRole('textbox', { name: label }) + await user.type(input, initial) + await user.click(input) + await user.keyboard('{Control>}a{/Control}0') + expect(input).toHaveValue('0') + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + await user.click(screen.getByRole('button', { name: 'Use default sources' })) + expect(input).toHaveValue('0') + expect(screen.getByRole('alert')).toHaveTextContent(`${label} must be a positive integer`) + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + expect(onChange.mock.calls.at(-1)?.[0]).toHaveProperty(key, initial === 'all' ? 'all' : 3) + await user.click(input) + await user.keyboard('{Control>}a{/Control}4') + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + }) + + it('clears only removed source errors and preserves other invalid fields', async () => { + const user = userEvent.setup() + const onChange = jest.fn() + render() + await user.type(screen.getByRole('combobox', { name: 'Add dataset' }), 'remote') + await user.click(screen.getByRole('button', { name: 'Add source' })) + await user.type(screen.getByRole('textbox', { name: 'remote limit' }), '0') + await user.type(screen.getByRole('textbox', { name: 'Max total' }), '0') + await user.click(screen.getByRole('button', { name: 'Remove remote' })) + expect(screen.queryByRole('textbox', { name: 'remote limit' })).not.toBeInTheDocument() + expect(screen.getByRole('alert')).toHaveTextContent('Max total must be a positive integer') + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + await user.clear(screen.getByRole('textbox', { name: 'Max total' })) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + await user.type(screen.getByRole('combobox', { name: 'Add dataset' }), 'remote') + await user.click(screen.getByRole('button', { name: 'Add source' })) + expect(screen.getByRole('textbox', { name: 'remote limit' })).toHaveValue('') + }) + + it('resets source drafts even when default sources have the same names', async () => { + const user = userEvent.setup() + render() + await user.type(screen.getByRole('textbox', { name: 'stored limit' }), '0') + expect(screen.getByRole('button', { name: 'Launch' })).toBeDisabled() + await user.click(screen.getByRole('button', { name: 'Use default sources' })) + expect(screen.getByRole('textbox', { name: 'stored limit' })).toHaveValue('') + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + }) + + it('replacing the sole selected source removes its invalid draft', async () => { + const user = userEvent.setup() + render() + await user.type(screen.getByRole('textbox', { name: 'stored limit' }), '0') + await user.click(screen.getByRole('combobox', { name: 'Add dataset' })) + await user.click(screen.getByRole('option', { name: 'remote' })) + await user.click(screen.getByRole('button', { name: 'Add source' })) + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(screen.getByRole('button', { name: 'Launch' })).toBeEnabled() + }) + +}) diff --git a/frontend/src/components/Scenarios/ScenarioDatasetTable.tsx b/frontend/src/components/Scenarios/ScenarioDatasetTable.tsx new file mode 100644 index 0000000000..ac23988494 --- /dev/null +++ b/frontend/src/components/Scenarios/ScenarioDatasetTable.tsx @@ -0,0 +1,206 @@ +import { useEffect, useState } from 'react' + +import { Button, Combobox, Field, Input, MessageBar, MessageBarBody, Option, Select, Text } from '@fluentui/react-components' + +import type { + DatasetInfo, ScenarioDatasetLimit, ScenarioDatasetOptions, ScenarioDatasetRequest, + ScenarioDatasetSettings, ScenarioDatasetSourceInput, +} from '@/types' + +import { useScenarioDatasetTableStyles } from './ScenarioDatasetTable.styles' + +interface ScenarioDatasetTableProps { + request: ScenarioDatasetRequest + defaults: ScenarioDatasetSettings + effective: ScenarioDatasetSettings + options: ScenarioDatasetOptions + datasets: DatasetInfo[] + disabled: boolean + loading: boolean + metadataError: string | null + onReload: () => void + onChange: (request: ScenarioDatasetRequest) => void + onValidationChange: (error: string | null) => void +} + +interface DatasetLimitFieldProps { + label: string + text: string + effective: number | 'all' + disabled: boolean + onChange: (text: string) => void +} + +function parseLimit(text: string): ScenarioDatasetLimit | null { + const value = text.trim().toLowerCase() + if (value === '' || value === 'default') return 'default' + if (value === 'all') return 'all' + const number = Number(value) + return /^\d+$/.test(value) && Number.isSafeInteger(number) && number > 0 ? number : null +} + +function DatasetLimitField({ label, text, effective, disabled, onChange }: DatasetLimitFieldProps) { + const styles = useScenarioDatasetTableStyles() + return ( + + onChange(data.value)} /> + + ) +} + +/** Displays effective settings while emitting only user edits, never catalog defaults. */ +export default function ScenarioDatasetTable({ + request, defaults, effective, options, datasets, disabled, loading, metadataError, + onReload, onChange, onValidationChange, +}: ScenarioDatasetTableProps) { + const styles = useScenarioDatasetTableStyles() + const [newName, setNewName] = useState('') + const [drafts, setDrafts] = useState>({}) + const [selectionError, setSelectionError] = useState(null) + const selected: ScenarioDatasetSourceInput[] = request.sources ?? effective.sources.map((source) => ({ name: source.name })) + const canSelect = options.supports_overrides && options.mode !== 'fixed' + const canEdit = options.supports_overrides && options.source_settings !== false + const candidates = datasets.filter((dataset) => options.mode === 'any' || options.allowed_names.includes(dataset.name)) + const limitText = (key: string, value: ScenarioDatasetLimit | null | undefined): string => + drafts[key] ?? (value == null || value === 'default' ? '' : String(value)) + const updateLimit = (key: string, text: string, apply: (value: ScenarioDatasetLimit) => void): void => { + setDrafts((previous) => ({ ...previous, [key]: text })) + const value = parseLimit(text) + if (value !== null) apply(value) + } + const clearSourceDrafts = (name?: string): void => { + setDrafts((previous) => Object.fromEntries(Object.entries(previous).filter(([key]) => + name === undefined ? !key.startsWith('source:') : key !== `source:${name}`, + ))) + setSelectionError(null) + } + const updateSource = (name: string, edits: Partial): void => { + onChange({ ...request, sources: selected.map((source) => source.name === name ? { ...source, ...edits } : source) }) + } + const updateGlobal = (key: 'max_per_dataset' | 'max_total', value: ScenarioDatasetLimit): void => { + const edits = { ...request } + if (value === 'default') delete edits[key] + else edits[key] = value + onChange(edits) + } + const invalidNames = selected.filter((source) => + options.mode === 'subset' && !options.allowed_names.includes(source.name), + ) + const limits: Array<[string, string]> = [] + if (options.supports_overrides) { + if (options.per_dataset_limit) limits.push(['Max per dataset', limitText('perDataset', request.max_per_dataset)]) + if (options.total_limit) limits.push(['Max total', limitText('total', request.max_total)]) + if (canEdit && options.per_dataset_limit) { + for (const source of selected) { + limits.push([`${source.name} limit`, limitText(`source:${source.name}`, source.max_size)]) + } + } + } + const errors = limits.filter(([, text]) => parseLimit(text) === null) + .map(([label]) => `${label} must be a positive integer, all, or default.`) + if (invalidNames.length > 0) { + errors.push(`Invalid selection for current parameters: ${invalidNames.map((source) => source.name).join(', ')}`) + } + if (selectionError) errors.push(selectionError) + const validationError = errors[0] ?? null + useEffect(() => { onValidationChange(validationError) }, [validationError, onValidationChange]) + return ( +
+ Datasets + + {metadataError && {metadataError}} + {errors.map((error) => {error})} + {options.supports_overrides &&
+ {options.per_dataset_limit && updateLimit('perDataset', text, (value) => updateGlobal('max_per_dataset', value))} />} + {options.total_limit && updateLimit('total', text, (value) => updateGlobal('max_total', value))} />} + + + +
} +
+ + {['Name', 'Stored origins', 'Stored examples', 'Limit', 'Fetch policy', 'Selection'].map((label) => + )} + {selected.map((source) => { + const metadata = datasets.find((dataset) => dataset.name === source.name) + const settings = effective.sources.find((item) => item.name === source.name) + ?? defaults.sources.find((item) => item.name === source.name) + const hasProvider = metadata?.provider_available || settings?.configured_provider + return + + + + + + + + })} +
{label}
{source.name}{metadata?.origins?.join(', ') || 'Unknown'} +
{hasProvider ? 'Provider available' : metadata ? 'Memory only' : 'Availability unknown'}
{metadata?.logical_examples ?? 'Unknown'}{canEdit && options.per_dataset_limit + ? updateLimit(`source:${source.name}`, text, (value) => updateSource(source.name, { max_size: value }))} /> + : settings?.max_size ?? 'all'}{canEdit + ? : settings?.fetch ?? effective.fetch}{canSelect && }
+
+ {canSelect &&
+ + setNewName(event.target.value)} + onOptionSelect={(_, data) => setNewName(data.optionValue ?? '')}> + {candidates.map((dataset) => )} + + + + +
} +
+ ) +} diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index c3ab93e110..87b100d47e 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -7,6 +7,7 @@ import { scenariosApi, targetsApi } from '@/services/api' import { useRuntime } from '@/hooks/useRuntime' import type { RegisteredScenario, + ScenarioRunSizeEstimateRequest, ScenarioRunSizeEstimateResponse, TargetInstance, } from '@/types' @@ -16,6 +17,7 @@ import ScenarioDetail from './ScenarioDetail' jest.mock('@/hooks/useRuntime', () => ({ useRuntime: jest.fn() })) jest.mock('@/services/api', () => ({ + datasetsApi: { listDatasets: jest.fn().mockResolvedValue({ items: [] }) }, scenariosApi: { estimateRun: jest.fn(), getScenario: jest.fn(), @@ -306,8 +308,9 @@ describe('ScenarioDetail', () => { await flushRenderedPromises() await user.type(screen.getByLabelText(/^iterations/), '1.5') await user.type(screen.getByLabelText(/^note/), 'Keep this value') - await user.type(screen.getByLabelText('Dataset override'), 'stale-dataset') - await user.type(screen.getByLabelText('Max dataset size'), '0') + await user.type(screen.getByRole('combobox', { name: 'Add dataset' }), 'stale-dataset') + await user.click(screen.getByRole('button', { name: 'Add source' })) + await user.type(screen.getByRole('textbox', { name: 'Max total' }), '0') await user.type(screen.getByLabelText('Harm categories'), 'stale-harm') await user.type(screen.getByLabelText('Data types'), 'stale-type') for (const label of ['Max concurrency', 'Max retries']) { @@ -316,8 +319,8 @@ describe('ScenarioDetail', () => { await user.type(input, '500') await user.tab() } - await user.click(screen.getByRole('button', { name: 'Launch scan' })) - expect(screen.getByRole('alert')).toHaveTextContent('iterations must be an integer.') + expect(screen.getByRole('button', { name: 'Launch scan' })).toBeDisabled() + expect(screen.getByRole('alert')).toHaveTextContent('Max total must be a positive integer') mockGetScenario.mockResolvedValue(makeScenario({ supported_parameter_names: ['note'], @@ -476,7 +479,7 @@ describe('ScenarioDetail', () => { expect(within(screen.getByRole('region', { name: 'Objective Target' })).queryByRole( 'combobox', { name: 'Adversarial Target' }, )).not.toBeInTheDocument() - expect(fallback.compareDocumentPosition(within(parameters).getByRole('textbox', { name: 'Dataset override' }))) + expect(fallback.compareDocumentPosition(within(parameters).getByRole('table', { name: 'Selected datasets' }))) .toBe(Node.DOCUMENT_POSITION_FOLLOWING) expect(fallback).toHaveAccessibleDescription('The registered target this scenario uses to generate attacks.') expect(fallback).toHaveValue('adversary') @@ -1083,10 +1086,8 @@ describe('ScenarioDetail', () => { expect(await screen.findByRole('heading', { name: 'Parameters' })).toBeInTheDocument() expect(screen.queryByText('Advanced options')).not.toBeInTheDocument() - expect(screen.getByTestId('max-dataset-size-input')).toHaveValue(8) - expect(screen.getByText( - 'The scenario default is 8. Edit it to override the default.', - )).toBeInTheDocument() + expect(screen.getByRole('textbox', { name: 'Max total' })).toHaveValue('') + expect(screen.getByText('Default/effective: 8. Blank or default retains the scenario setting.')).toBeInTheDocument() const estimate = screen.getByTestId('run-estimate') expect(within(estimate).getByText('Dataset size')).toBeInTheDocument() expect(within(estimate).getByText('8')).toBeInTheDocument() @@ -1116,7 +1117,8 @@ describe('ScenarioDetail', () => { renderDetail('/scanner/foundry.red_team_agent') - expect(await screen.findByTestId('max-dataset-size-input')).toHaveValue(5) + expect(await screen.findByRole('textbox', { name: 'Max total' })).toHaveValue('') + expect(screen.getByText('Default/effective: 5. Blank or default retains the scenario setting.')).toBeInTheDocument() expect(await screen.findByText('Up to 10')).toBeInTheDocument() expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') }) @@ -1130,7 +1132,7 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - expect(await screen.findByRole('spinbutton', { name: 'Max dataset size' })).toHaveValue(null) + expect(await screen.findByRole('textbox', { name: 'Max total' })).toHaveValue('') await waitFor(() => expect(mockEstimateRun).toHaveBeenCalled()) expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size') }) @@ -1144,9 +1146,8 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - const input = await screen.findByRole('spinbutton', { name: 'Max dataset size' }) - expect(input).toBeDisabled() - expect(input).toHaveValue(null) + await screen.findByRole('table', { name: 'Selected datasets' }) + expect(screen.queryByRole('textbox', { name: 'Max total' })).not.toBeInTheDocument() expect(screen.queryByRole('button', { name: 'Use all data' })).not.toBeInTheDocument() }) @@ -1160,8 +1161,8 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - const input = await screen.findByRole('spinbutton', { name: 'Max dataset size' }) - expect(input).toHaveValue(5) + const input = await screen.findByRole('textbox', { name: 'Max total' }) + await user.type(input, '2') await user.clear(input) expect(screen.queryByRole('button', { name: 'Use all data' })).not.toBeInTheDocument() await waitFor(() => expect(mockEstimateRun).toHaveBeenCalled()) @@ -1174,13 +1175,12 @@ describe('ScenarioDetail', () => { it('sends an explicit all total for both estimation and launch', async () => { const user = userEvent.setup() renderDetail('/scanner/foundry.red_team_agent') - const unlimited = await screen.findByRole('checkbox', { name: 'No total dataset limit' }) - await user.click(unlimited) - expect(screen.getByRole('spinbutton', { name: 'Max dataset size' })).toBeDisabled() - await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('max_dataset_size', 'all')) + const input = await screen.findByRole('textbox', { name: 'Max total' }) + await user.type(input, 'all') + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('datasets.max_total', 'all')) await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) - expect(mockStartRun.mock.calls[0][0]).toHaveProperty('max_dataset_size', 'all') + expect(mockStartRun.mock.calls[0][0]).toHaveProperty('datasets.max_total', 'all') }) it('restores the scenario default when no-total-limit is unchecked', async () => { @@ -1192,11 +1192,11 @@ describe('ScenarioDetail', () => { }, })) renderDetail('/scanner/foundry.red_team_agent') - const unlimited = await screen.findByRole('checkbox', { name: 'No total dataset limit' }) - await user.click(unlimited) - await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('max_dataset_size', 'all')) - await user.click(unlimited) - expect(screen.getByRole('spinbutton', { name: 'Max dataset size' })).toHaveValue(5) + const input = await screen.findByRole('textbox', { name: 'Max total' }) + await user.type(input, 'all') + await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toHaveProperty('datasets.max_total', 'all')) + await user.clear(input) + expect(input).toHaveValue('') await waitFor(() => expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('max_dataset_size')) await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) @@ -1208,17 +1208,22 @@ describe('ScenarioDetail', () => { renderDetail('/scanner/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.type(screen.getByTestId('dataset-override-input'), 'ds_a, ds_b') - await user.type(screen.getByTestId('max-dataset-size-input'), '25') + const addDataset = screen.getByRole('combobox', { name: 'Add dataset' }) + await user.type(addDataset, 'ds_a') + await user.click(screen.getByRole('button', { name: 'Add source' })) + await user.type(addDataset, 'ds_b') + await user.click(screen.getByRole('button', { name: 'Add source' })) + await user.click(screen.getByRole('button', { name: 'Remove harmbench' })) + await user.type(screen.getByRole('textbox', { name: 'Max total' }), '25') await user.type(screen.getByTestId('harm-categories-filter-input'), 'cyber, violence') await user.type(screen.getByTestId('data-types-filter-input'), 'text, image_path') await confirmRunPreview(user) await waitFor(() => expect(mockStartRun).toHaveBeenCalled()) const request = mockStartRun.mock.calls[0][0] - expect(request.dataset_names).toEqual(['ds_a', 'ds_b']) - expect(request.max_dataset_size).toBe(25) - expect(request.dataset_filters).toEqual({ + expect(request.datasets.sources).toEqual([{ name: 'ds_a' }, { name: 'ds_b' }]) + expect(request.datasets.max_total).toBe(25) + expect(request.datasets.filters).toEqual({ harm_categories: ['cyber', 'violence'], data_types: ['text', 'image_path'], }) @@ -1227,11 +1232,13 @@ describe('ScenarioDetail', () => { expect.objectContaining({ target_name: 'target-a', techniques: ['default_technique'], - dataset_names: ['ds_a', 'ds_b'], - max_dataset_size: 25, - dataset_filters: { + datasets: { + sources: [{ name: 'ds_a' }, { name: 'ds_b' }], + max_total: 25, + filters: { harm_categories: ['cyber', 'violence'], data_types: ['text', 'image_path'], + }, }, include_baseline: true, }), @@ -1240,17 +1247,56 @@ describe('ScenarioDetail', () => { expect(mockEstimateRun.mock.calls.at(-1)?.[1]).not.toHaveProperty('labels') }) + it('refreshes parameter-aware defaults after invalid source edits without silently dropping them', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValueOnce(makeScenario({ + supported_parameters: [ + { name: 'scope', type_name: 'str', required: false, default: 'all', choices: null, is_list: false }, + ], + })) + mockEstimateRun.mockImplementation(async (_name: string, request: ScenarioRunSizeEstimateRequest) => { + const narrowed = request.scenario_params?.scope === 'one' + if (narrowed && request.datasets?.sources) { + throw { isAxiosError: true, response: { status: 400, data: { detail: 'Dataset is outside the selected scope.' } } } + } + const name = narrowed ? 'remote' : 'harmbench' + const settings = { + sources: [{ name, max_size: 5, fetch: 'never', configured_provider: false }], + max_per_dataset: 5, max_total: 5, fetch: 'never', filters: {}, + } + return { + ...makeEstimate(10), dataset_defaults: settings, effective_datasets: settings, + dataset_options: { + mode: 'subset', allowed_names: [name], per_dataset_limit: true, total_limit: true, + supports_overrides: true, filters: true, + }, + } + }) + renderDetail('/scanner/foundry.red_team_agent') + await screen.findByRole('textbox', { name: 'harmbench limit' }) + await user.type(screen.getByRole('textbox', { name: 'harmbench limit' }), '1') + await user.clear(screen.getByTestId('scenario-param-scope')) + await user.type(screen.getByTestId('scenario-param-scope'), 'one') + await screen.findByText('Invalid selection for current parameters: harmbench') + expect(screen.getByRole('textbox', { name: 'harmbench limit' })).toHaveValue('1') + expect(mockEstimateRun.mock.calls.some((call) => + call[1].scenario_params?.scope === 'one' && call[1].datasets === undefined, + )).toBe(true) + await user.click(screen.getByRole('button', { name: 'Use default sources' })) + await screen.findByRole('rowheader', { name: 'remote' }) + }) + it('rejects a non-positive-integer max dataset size', async () => { const user = userEvent.setup() renderDetail('/scanner/foundry.red_team_agent') await screen.findByTestId('scenario-target-select') - await user.type(screen.getByTestId('max-dataset-size-input'), '0') + await user.type(screen.getByRole('textbox', { name: 'Max total' }), '0') await user.click(screen.getByTestId('launch-scenario-btn')) - expect(await screen.findByRole('alert')).toHaveTextContent( - 'Max dataset size must be a positive integer.', - ) + expect(await screen.findAllByRole('alert')).toEqual(expect.arrayContaining([ + expect.objectContaining({ textContent: expect.stringContaining('Max total must be a positive integer, all, or default.') }), + ])) expect(mockStartRun).not.toHaveBeenCalled() }) diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index eb4cb137ba..0b05aa4550 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -32,6 +32,7 @@ import { Link, useNavigate, useParams } from 'react-router' import MarkdownContent from '@/components/Markdown/MarkdownContent' import TargetSelect from '@/components/Config/TargetSelect' import { useRuntime } from '@/hooks/useRuntime' +import { useScenarioDatasets } from '@/hooks/useScenarioDatasets' import ParameterField from '@/components/Parameters/ParameterField' import SingleStepSpinButton from '@/components/Parameters/SingleStepSpinButton' import { @@ -51,6 +52,10 @@ import type { ScenarioRunSizeEstimateRequest, ScenarioRunEstimateState, ScenarioTechniqueSummary, + ScenarioDatasetRequest, + ScenarioDatasetSettings, + ScenarioDatasetOptions, + ScenarioRunSizeEstimateResponse, TargetInstance, } from '@/types' import { routerPathParamValue, scenarioRunRoutePath } from '@/utils/routeParams' @@ -61,6 +66,7 @@ import { ScenarioRunEstimateDetails } from './ScenarioRunEstimate' import { normalizeScenarioMarkdown } from './scenarioMarkdown' import { mapScenarioRunEstimate } from './scenarioRunEstimateAdapter' import { techniqueSetName } from './scenarioTechniqueSets' +import ScenarioDatasetTable from './ScenarioDatasetTable' /** * Common/opaque parameters the base scenario declares via @@ -74,6 +80,7 @@ const COMMON_SCENARIO_PARAMETER_NAMES = new Set([ 'scenario_techniques', 'technique_converters', 'dataset_config', + 'dataset_overrides', 'memory_labels', 'max_concurrency', 'max_retries', @@ -99,11 +106,6 @@ function targetOptionLabel(target: TargetInstance): string { : target.target_registry_name } -function defaultMaxDatasetSize(scenario: RegisteredScenario): string { - const limit = scenario.default_run_size.dataset_limit - return limit.state === 'value' ? String(limit.value) : '' -} - /** Resolves a Fluent `SpinButton` change event to a numeric value, preferring the parsed `value` over the raw `displayValue`. */ function resolveSpinButtonValue(data: { value?: number | null; displayValue?: string }, previous: number): number { if (typeof data.value === 'number') { @@ -238,8 +240,8 @@ interface BuildEstimateRequestInput { techniques: string[] dynamicParameters: Parameter[] scenarioParamValues: Record - datasetOverride: string - maxDatasetSize: string + datasetEdits: ScenarioDatasetRequest + datasetError: string | null harmCategoriesFilter: string dataTypesFilter: string includeBaseline: boolean @@ -295,8 +297,8 @@ function buildEstimateRequest({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, - maxDatasetSize, + datasetEdits, + datasetError, harmCategoriesFilter, dataTypesFilter, includeBaseline, @@ -314,19 +316,8 @@ function buildEstimateRequest({ scenarioParams = result.parameters } - let maxDatasetSizeValue: number | 'all' | undefined const supportsDatasets = supportsParameter(scenario, 'dataset_config') - const trimmedMaxDatasetSize = supportsDatasets ? maxDatasetSize.trim() : '' - if (trimmedMaxDatasetSize === 'all') { - maxDatasetSizeValue = 'all' - } else if (trimmedMaxDatasetSize.length > 0) { - const parsed = Number(trimmedMaxDatasetSize) - if (!Number.isInteger(parsed) || parsed < 1) { - return { ok: false, error: 'Max dataset size must be a positive integer.' } - } - maxDatasetSizeValue = parsed - } - const datasetNames = supportsDatasets ? parseDatasetNames(datasetOverride) : [] + if (supportsDatasets && datasetError) return { ok: false, error: datasetError } const request: ScenarioRunSizeEstimateRequest = {} if (supportsParameter(scenario, 'scenario_techniques')) { request.techniques = techniques @@ -340,19 +331,15 @@ function buildEstimateRequest({ if (scenario.uses_default_adversarial_target && adversarialTargetName) { request.adversarial_target_name = adversarialTargetName } - if (datasetNames.length > 0) { - request.dataset_names = datasetNames - } - if (maxDatasetSizeValue !== undefined) { - request.max_dataset_size = maxDatasetSizeValue - } const harmCategories = supportsDatasets ? parseDatasetNames(harmCategoriesFilter) : [] const dataTypes = supportsDatasets ? parseDatasetNames(dataTypesFilter) : [] if (harmCategories.length > 0 || dataTypes.length > 0) { - request.dataset_filters = { + request.datasets = { ...datasetEdits, filters: { ...(harmCategories.length > 0 ? { harm_categories: harmCategories } : {}), ...(dataTypes.length > 0 ? { data_types: dataTypes } : {}), - } + } } + } else if (supportsDatasets && Object.keys(datasetEdits).length > 0) { + request.datasets = datasetEdits } if (scenarioParams) { request.scenario_params = scenarioParams @@ -628,12 +615,21 @@ function ScenarioLaunchForm({ const [baselineChecked, setBaselineChecked] = useState( () => !isBaselineForbidden && scenario.include_baseline_by_default, ) - const [datasetOverride, setDatasetOverride] = useState('') - const configuredDefaultMaxDatasetSize = useMemo( - () => defaultMaxDatasetSize(scenario), - [scenario], - ) - const [maxDatasetSize, setMaxDatasetSize] = useState(configuredDefaultMaxDatasetSize) + const [datasetEdits, setDatasetEdits] = useState({}) + const [datasetError, setDatasetError] = useState(null) + const datasetMetadata = useScenarioDatasets() + const [datasetProjection, setDatasetProjection] = useState(null) + const datasetDefaults: ScenarioDatasetSettings = datasetProjection?.dataset_defaults ?? scenario.dataset_defaults ?? { + sources: scenario.default_datasets.map((name) => ({ name, max_size: 'all', fetch: 'if_missing', configured_provider: false })), + max_per_dataset: 'all', max_total: scenario.default_run_size.dataset_limit.value ?? 'all', + fetch: 'if_missing', filters: {}, + } + const effectiveDatasets = datasetProjection?.effective_datasets ?? datasetDefaults + const datasetOptions: ScenarioDatasetOptions = datasetProjection?.dataset_options ?? scenario.dataset_options ?? { + mode: 'any', allowed_names: [], per_dataset_limit: true, + total_limit: scenario.default_run_size.dataset_limit.state !== 'not_applicable', + supports_overrides: true, filters: true, + } const [harmCategoriesFilter, setHarmCategoriesFilter] = useState('') const [dataTypesFilter, setDataTypesFilter] = useState('') const [maxConcurrency, setMaxConcurrency] = useState(DEFAULT_MAX_CONCURRENCY) @@ -686,15 +682,8 @@ function ScenarioLaunchForm({ [isBaselineForbidden, supportsBaseline, supportsTechniques, techniqueOptions], ) const techniques = selectedTechniques - const maxDatasetSizeOverride = maxDatasetSize.trim() - && maxDatasetSize !== configuredDefaultMaxDatasetSize - ? maxDatasetSize - : '' - const datasetSizeLabel = scenario.default_run_size.dataset_limit.state === 'not_applicable' - ? 'Not applicable' - : maxDatasetSize === 'all' - ? 'No total limit' - : maxDatasetSize.trim() || configuredDefaultMaxDatasetSize || 'Scenario default' + const datasetSizeLabel = !datasetOptions.total_limit ? 'Not applicable' + : effectiveDatasets.max_total === 'all' ? 'No total limit' : String(effectiveDatasets.max_total) const estimateResult = useMemo( () => buildEstimateRequest({ scenario, @@ -703,8 +692,8 @@ function ScenarioLaunchForm({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, - maxDatasetSize: maxDatasetSizeOverride, + datasetEdits, + datasetError, harmCategoriesFilter, dataTypesFilter, includeBaseline: isBaselineForbidden ? false : baselineChecked, @@ -712,12 +701,12 @@ function ScenarioLaunchForm({ [ adversarialTargetName, baselineChecked, - datasetOverride, + datasetEdits, + datasetError, dataTypesFilter, dynamicParameters, harmCategoriesFilter, isBaselineForbidden, - maxDatasetSizeOverride, scenario, scenarioParamValues, targetName, @@ -732,8 +721,8 @@ function ScenarioLaunchForm({ techniques, dynamicParameters, scenarioParamValues, - datasetOverride, - maxDatasetSize: maxDatasetSizeOverride, + datasetEdits, + datasetError, harmCategoriesFilter, dataTypesFilter, maxConcurrency, @@ -744,14 +733,14 @@ function ScenarioLaunchForm({ [ adversarialTargetName, baselineChecked, - datasetOverride, + datasetEdits, + datasetError, dataTypesFilter, dynamicParameters, harmCategoriesFilter, isBaselineForbidden, labels, maxConcurrency, - maxDatasetSizeOverride, maxRetries, scenario, scenarioParamValues, @@ -790,6 +779,7 @@ function ScenarioLaunchForm({ return } const result = mapScenarioRunEstimate(response, 'request') + setDatasetProjection(response) setEstimateRequestState({ status: 'resolved', requestKey: estimateRequestKey, @@ -806,11 +796,26 @@ function ScenarioLaunchForm({ ) { return } + const error = toApiError(err) setEstimateRequestState({ status: 'error', requestKey: estimateRequestKey, - error: toApiError(err).detail, + error: error.detail, }) + if (estimateRequest.datasets && (error.status === 400 || error.status === 422)) { + const { datasets, ...defaultRequest } = estimateRequest + void datasets + // Keep invalid edits visible while refreshing parameter-aware allowed sources. + void scenariosApi.estimateRun(scenario.scenario_name, defaultRequest, controller.signal) + .then((response) => { + if (controller.signal.aborted || requestSequence !== estimateSequenceRef.current) return + setDatasetProjection((previous) => ({ + ...response, + effective_datasets: previous?.effective_datasets ?? response.effective_datasets, + })) + }) + .catch(() => { /* The original validation error remains authoritative. */ }) + } }) }, ESTIMATE_DEBOUNCE_MS) @@ -924,6 +929,7 @@ function ScenarioLaunchForm({ try { const summary = await scenariosApi.startRun(requestResult.request) + datasetMetadata.reload() setPreviewOpen(false) navigate(scenarioRunRoutePath(summary.scenario_result_id), { state: { scenarioName: scenario.scenario_name }, @@ -952,8 +958,8 @@ function ScenarioLaunchForm({ const displayedEstimate = estimateFromState(estimateState) const selectedTechniqueCount = (supportsTechniques ? selectedTechniques.length : 0) + (supportsBaseline && baselineChecked && !isBaselineForbidden ? 1 : 0) - const previewDatasets = parseDatasetNames(datasetOverride) - const effectiveDatasets = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets + const previewDatasets = (datasetEdits.sources ?? effectiveDatasets.sources).map((source) => source.name) + const previewDatasetNames = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets const previewHarmCategories = parseDatasetNames(harmCategoriesFilter) const previewDataTypes = parseDatasetNames(dataTypesFilter) @@ -1155,45 +1161,12 @@ function ScenarioLaunchForm({ )} {supportsDatasets && ( <> - - setDatasetOverride(data.value)} - placeholder={scenario.default_datasets.join(', ') || undefined} - data-testid="dataset-override-input" - /> - - - setMaxDatasetSize(data.value)} - data-testid="max-dataset-size-input" - /> - - setMaxDatasetSize(data.checked === true ? 'all' : configuredDefaultMaxDatasetSize)} + - Per-dataset limits still apply. An empty input uses the scenario default total. setHarmCategoriesFilter(data.value)} data-testid="harm-categories-filter-input" @@ -1214,7 +1187,7 @@ function ScenarioLaunchForm({ setDataTypesFilter(data.value)} data-testid="data-types-filter-input" @@ -1319,7 +1292,7 @@ function ScenarioLaunchForm({ className={styles.launchButton} appearance="primary" type="submit" - disabled={!runtime.ready || !defaultsReady || staleSelection || unavailableSelection || submitting || techniqueSelectionInvalid} + disabled={!runtime.ready || !defaultsReady || staleSelection || unavailableSelection || submitting || techniqueSelectionInvalid || (supportsDatasets && datasetError !== null)} data-testid="launch-scenario-btn" > Launch scan @@ -1373,8 +1346,8 @@ function ScenarioLaunchForm({
- {effectiveDatasets.length > 0 - ? effectiveDatasets.join(', ') + {previewDatasetNames.length > 0 + ? previewDatasetNames.join(', ') : 'No datasets declared'} @@ -1444,7 +1417,7 @@ function ScenarioLaunchForm({