From d6e5598c868a4e67a62179f319c0950383cf82da Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sat, 10 Oct 2026 19:43:53 -0700 Subject: [PATCH 1/3] FIX: Honor scenario input support in scanner forms Expose all declared parameter names and gate controls, validation, and request aliases. Preserve omitted common inputs across launch and resume. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c198c14a-c4c7-4e5b-b068-1e62287c0d92 --- frontend/README.md | 13 ++ frontend/e2e/numeric-controls.spec.ts | 4 + frontend/e2e/onboarding-tour.spec.ts | 4 + frontend/e2e/scenario-history.spec.ts | 5 + frontend/src/App.labels.test.tsx | 4 + .../Scenarios/ScenarioCatalog.test.tsx | 1 + .../Scenarios/ScenarioDetail.test.tsx | 141 +++++++++++++++ .../components/Scenarios/ScenarioDetail.tsx | 163 +++++++++++------- .../Scenarios/ScenarioFlow.test.tsx | 4 + frontend/src/services/api.test.ts | 1 + frontend/src/types/index.ts | 4 +- .../backend/services/scenario_run_service.py | 61 +++++-- pyrit/backend/services/scenario_service.py | 1 + pyrit/models/catalog/scenario.py | 15 +- .../unit/backend/test_scenario_run_service.py | 69 ++++++++ tests/unit/backend/test_scenario_service.py | 26 +++ tests/unit/cli/test_api_client.py | 1 + tests/unit/cli/test_output.py | 1 + tests/unit/cli/test_pyrit_scan.py | 5 + tests/unit/cli/test_pyrit_shell.py | 2 + 20 files changed, 439 insertions(+), 86 deletions(-) diff --git a/frontend/README.md b/frontend/README.md index 4164661c99..aaf9bf3372 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -19,6 +19,19 @@ Failed runs offer **Resume run** on the run page and **Resume** in Scanner Histo See the [GUI guide](../doc/gui/0_gui.md#resuming-a-failed-scanner-run) for resume behavior and the saved launch configuration requirement. +## Supported scenario inputs + +Scanner forms show only inputs declared by the selected scenario. Hidden +inputs are not validated or sent in estimate or launch requests, even if an +older draft contains values for them. Dataset overrides, size limits, and +filters are available only when the scenario declares `dataset_config`. + +The catalog's required `supported_parameter_names` field includes all declared +names, including opaque inputs with dedicated controls. `supported_parameters` +contains only typed external input descriptors. Defaults and run-size estimates +do not determine input support. A catalog response without the names field +shows an error and blocks the form; update the backend and reload. + ## Default labels after runtime changes When the runtime configuration changes, CoPyRIT reloads the server's default diff --git a/frontend/e2e/numeric-controls.spec.ts b/frontend/e2e/numeric-controls.spec.ts index 09f688297c..a9b7cc8705 100644 --- a/frontend/e2e/numeric-controls.spec.ts +++ b/frontend/e2e/numeric-controls.spec.ts @@ -25,6 +25,10 @@ const SCENARIO: RegisteredScenario = { { name: 'iterations', type_name: 'int', required: false, default: '2', choices: null, is_list: false }, { name: 'temperature', type_name: 'float', required: false, default: '0.5', choices: null, is_list: false }, ], + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', 'iterations', 'temperature', + ], default_run_size: { dataset_size: { kind: 'indeterminate', reason: 'configuration_unavailable', detail: 'Population configuration is not available.' }, dataset_limit: { state: 'scenario_default' }, diff --git a/frontend/e2e/onboarding-tour.spec.ts b/frontend/e2e/onboarding-tour.spec.ts index 3572c48193..dae177aaa3 100644 --- a/frontend/e2e/onboarding-tour.spec.ts +++ b/frontend/e2e/onboarding-tour.spec.ts @@ -22,6 +22,10 @@ const TOUR_SCENARIO: RegisteredScenario = { baseline_policy: "disabled", include_baseline_by_default: false, supported_parameters: [], + supported_parameter_names: [ + "objective_target", "scenario_techniques", "dataset_config", "memory_labels", + "max_concurrency", "max_retries", "include_baseline", + ], default_run_size: { status: "unavailable", dataset_size: { kind: "indeterminate", detail: "Mock estimate only." }, diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts index 23f2e4c9a1..1abb526004 100644 --- a/frontend/e2e/scenario-history.spec.ts +++ b/frontend/e2e/scenario-history.spec.ts @@ -91,6 +91,11 @@ const catalogScenario = { baseline_policy: "enabled", include_baseline_by_default: false, uses_default_adversarial_target: false, + supported_parameter_names: [ + "objective_target", "scenario_techniques", "dataset_config", "memory_labels", + "max_concurrency", "max_retries", "include_baseline", "num_jailbreaks", + "num_jailbreak_attempts", "jailbreak_names", + ], supported_parameters: [ { name: "num_jailbreaks", diff --git a/frontend/src/App.labels.test.tsx b/frontend/src/App.labels.test.tsx index c2ad8fa90e..9f17d1610f 100644 --- a/frontend/src/App.labels.test.tsx +++ b/frontend/src/App.labels.test.tsx @@ -77,6 +77,10 @@ const SCENARIO: RegisteredScenario = { baseline_policy: 'forbidden', include_baseline_by_default: false, supported_parameters: [], + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', + ], uses_default_adversarial_target: false, default_run_size: { dataset_size: { kind: 'bounded', value: 1 }, diff --git a/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx b/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx index 84e19457c1..7bbb1b639f 100644 --- a/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx @@ -66,6 +66,7 @@ function makeScenario(overrides: Partial & { scenario_name: include_baseline_by_default: true, uses_default_adversarial_target: true, supported_parameters: [], + supported_parameter_names: [], default_run_size: { dataset_size: { kind: 'indeterminate', detail: 'Population configuration is not available.' }, dataset_limit: { state: 'scenario_default' }, diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index 477c78c8fb..c3ab93e110 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -4,6 +4,7 @@ import { FluentProvider, webLightTheme } from '@fluentui/react-components' import { MemoryRouter, Route, Routes } from 'react-router' import { scenariosApi, targetsApi } from '@/services/api' +import { useRuntime } from '@/hooks/useRuntime' import type { RegisteredScenario, ScenarioRunSizeEstimateResponse, @@ -12,6 +13,8 @@ import type { import ScenarioDetail from './ScenarioDetail' +jest.mock('@/hooks/useRuntime', () => ({ useRuntime: jest.fn() })) + jest.mock('@/services/api', () => ({ scenariosApi: { estimateRun: jest.fn(), @@ -27,6 +30,7 @@ const mockGetScenario = scenariosApi.getScenario as jest.Mock const mockEstimateRun = scenariosApi.estimateRun as jest.Mock const mockStartRun = scenariosApi.startRun as jest.Mock const mockListTargets = targetsApi.listTargets as jest.Mock +const mockUseRuntime = jest.mocked(useRuntime) const mockNavigate = jest.fn() const RAW_IMAGE_HTML = ['<', 'img src=x onerror="alert(1)">'].join('') @@ -63,6 +67,11 @@ function makeScenario(overrides: Partial = {}): RegisteredSc include_baseline_by_default: true, uses_default_adversarial_target: true, supported_parameters: [], + supported_parameter_names: overrides.supported_parameter_names ?? [ + 'objective_target', 'scenario_techniques', 'technique_converters', 'dataset_config', + 'memory_labels', 'max_concurrency', 'max_retries', 'include_baseline', + ...(overrides.supported_parameters ?? []).map((parameter) => parameter.name), + ], default_run_size: { dataset_size: { kind: 'indeterminate', detail: 'Population configuration is not available.' }, dataset_limit: { state: 'scenario_default' }, @@ -190,6 +199,7 @@ function renderDetail( describe('ScenarioDetail', () => { beforeEach(() => { jest.clearAllMocks() + mockUseRuntime.mockReturnValue({ ready: true, state: 'ready', generation: 'first' }) mockGetScenario.mockReset() mockEstimateRun.mockReset() mockListTargets.mockReset() @@ -203,6 +213,137 @@ describe('ScenarioDetail', () => { jest.useRealTimers() }) + it('blocks a legacy catalog response without support names instead of guessing common support', async () => { + const legacyScenario = makeScenario() + Reflect.deleteProperty(legacyScenario, 'supported_parameter_names') + mockGetScenario.mockResolvedValue(legacyScenario) + + renderDetail('/scanner/foundry.red_team_agent') + + expect(await screen.findByText(/catalog has no input support metadata/i)).toBeInTheDocument() + expect(screen.queryByRole('form')).not.toBeInTheDocument() + expect(mockEstimateRun).not.toHaveBeenCalled() + expect(mockStartRun).not.toHaveBeenCalled() + }) + + it.each([ + ['dataset_config', 'Dataset override', ['dataset_names', 'max_dataset_size', 'dataset_filters']], + ['scenario_techniques', 'default_technique', ['techniques']], + ['include_baseline', 'baseline', ['include_baseline']], + ['max_concurrency', 'Max concurrency', ['max_concurrency']], + ['max_retries', 'Max retries', ['max_retries']], + ['memory_labels', null, ['labels']], + ['objective_target', 'Objective Target', ['target_name']], + ])('omits controls and request aliases for undeclared %s', async ( + name: string, label: string | null, requestFields: string[], + ) => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const scenario = makeScenario() + scenario.supported_parameter_names = scenario.supported_parameter_names.filter( + (parameterName) => parameterName !== name, + ) + // Defaults and estimate data do not establish support for an input. + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockResolvedValue(makeEstimate(8)) + renderDetail('/scanner/foundry.red_team_agent') + await flushRenderedPromises() + await advanceTimers(300) + + if (label) { + expect(screen.queryByLabelText(label)).not.toBeInTheDocument() + } + if (name === 'dataset_config') { + expect(screen.queryByLabelText('Max dataset size')).not.toBeInTheDocument() + expect(screen.queryByLabelText('Harm categories')).not.toBeInTheDocument() + expect(screen.queryByLabelText('Data types')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox', { name: 'No total dataset limit' })).not.toBeInTheDocument() + } + expect(mockEstimateRun).toHaveBeenCalledTimes(1) + const preview = await openRunPreview(user) + if (name === 'dataset_config') { + expect(within(preview).queryByText('Datasets')).not.toBeInTheDocument() + expect(within(preview).queryByText('Dataset filters')).not.toBeInTheDocument() + } + await user.click(screen.getByTestId('confirm-launch-scenario-btn')) + expect(mockStartRun).toHaveBeenCalledTimes(1) + for (const field of requestFields) { + expect(mockEstimateRun.mock.calls[0][1]).not.toHaveProperty(field) + expect(mockStartRun.mock.calls[0][0]).not.toHaveProperty(field) + } + }) + + it('keeps baseline selection independent when attack technique selection is unsupported', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(makeScenario({ + supported_parameter_names: ['objective_target', 'include_baseline'], + all_techniques: [], + default_techniques: [], + })) + renderDetail('/scanner/foundry.red_team_agent') + const baseline = await screen.findByRole('checkbox', { name: 'baseline' }) + await user.click(baseline) + await confirmRunPreview(user) + + expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: 'foundry.red_team_agent', + target_name: 'target-a', + include_baseline: false, + }) + expect(screen.queryByText('Select at least one attack technique.')).not.toBeInTheDocument() + }) + + it('drops unsupported invalid drafts after catalog refresh and preserves supported values', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const parameters = [ + { name: 'iterations', type_name: 'int', required: true, default: null, choices: null, is_list: false }, + { name: 'note', type_name: 'str', required: true, default: null, choices: null, is_list: false }, + ] + mockGetScenario.mockResolvedValue(makeScenario({ supported_parameters: parameters })) + mockEstimateRun.mockResolvedValue(makeEstimate(8)) + const detail = renderDetail('/scanner/foundry.red_team_agent') + await flushRenderedPromises() + await user.type(screen.getByLabelText(/^iterations/), '1.5') + await user.type(screen.getByLabelText(/^note/), 'Keep this value') + await user.type(screen.getByLabelText('Dataset override'), 'stale-dataset') + await user.type(screen.getByLabelText('Max dataset size'), '0') + await user.type(screen.getByLabelText('Harm categories'), 'stale-harm') + await user.type(screen.getByLabelText('Data types'), 'stale-type') + for (const label of ['Max concurrency', 'Max retries']) { + const input = screen.getByRole('spinbutton', { name: label }) + await user.clear(input) + await user.type(input, '500') + await user.tab() + } + await user.click(screen.getByRole('button', { name: 'Launch scan' })) + expect(screen.getByRole('alert')).toHaveTextContent('iterations must be an integer.') + + mockGetScenario.mockResolvedValue(makeScenario({ + supported_parameter_names: ['note'], + // A removed external descriptor must not validate or submit a stale value. + supported_parameters: parameters, + uses_default_adversarial_target: false, + })) + mockUseRuntime.mockReturnValue({ ready: true, state: 'ready', generation: 'second' }) + detail.updateDefaults({ targets: [] }) + await flushRenderedPromises() + await user.click(screen.getByRole('button', { name: 'I have reviewed the refreshed selections' })) + await advanceTimers(300) + + expect(screen.getByLabelText(/^note/)).toHaveValue('Keep this value') + expect(screen.queryByLabelText(/^iterations/)).not.toBeInTheDocument() + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toEqual({ + scenario_params: { note: 'Keep this value' }, + }) + await confirmRunPreview(user) + expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: 'foundry.red_team_agent', + scenario_params: { note: 'Keep this value' }, + }) + }) + it('shows a loading state while fetching', () => { mockGetScenario.mockReturnValue(new Promise(() => {})) renderDetail('/scanner/foundry.red_team_agent') diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index f4378c4f3c..eb4cb137ba 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -63,8 +63,8 @@ import { mapScenarioRunEstimate } from './scenarioRunEstimateAdapter' import { techniqueSetName } from './scenarioTechniqueSets' /** - * Common/opaque parameters every scenario declares via - * `Scenario._common_scenario_parameters` — the launch form already exposes a + * Common/opaque parameters the base scenario declares via + * `Scenario._common_scenario_parameters` — the launch form exposes a * purpose-built control for each of these (target, techniques, datasets, * labels, concurrency, retries, baseline), and `technique_converters` has no * UI at all. They're hidden from the dynamic scenario-specific parameter list. @@ -88,6 +88,10 @@ const DEFAULT_MAX_CONCURRENCY = 10 const DEFAULT_MAX_RETRIES = 0 const ESTIMATE_DEBOUNCE_MS = 300 +function supportsParameter(scenario: RegisteredScenario, name: string): boolean { + return scenario.supported_parameter_names.includes(name) +} + function targetOptionLabel(target: TargetInstance): string { const modelName = targetModelName(target) return modelName @@ -297,7 +301,7 @@ function buildEstimateRequest({ dataTypesFilter, includeBaseline, }: BuildEstimateRequestInput): BuildEstimateRequestResult { - if (techniques.length === 0) { + if (supportsParameter(scenario, 'scenario_techniques') && techniques.length === 0) { return { ok: false, error: 'Select at least one technique.' } } @@ -311,7 +315,8 @@ function buildEstimateRequest({ } let maxDatasetSizeValue: number | 'all' | undefined - const trimmedMaxDatasetSize = maxDatasetSize.trim() + const supportsDatasets = supportsParameter(scenario, 'dataset_config') + const trimmedMaxDatasetSize = supportsDatasets ? maxDatasetSize.trim() : '' if (trimmedMaxDatasetSize === 'all') { maxDatasetSizeValue = 'all' } else if (trimmedMaxDatasetSize.length > 0) { @@ -321,12 +326,15 @@ function buildEstimateRequest({ } maxDatasetSizeValue = parsed } - const datasetNames = parseDatasetNames(datasetOverride) - const request: ScenarioRunSizeEstimateRequest = { - techniques, - include_baseline: includeBaseline, + const datasetNames = supportsDatasets ? parseDatasetNames(datasetOverride) : [] + const request: ScenarioRunSizeEstimateRequest = {} + if (supportsParameter(scenario, 'scenario_techniques')) { + request.techniques = techniques + } + if (supportsParameter(scenario, 'include_baseline')) { + request.include_baseline = includeBaseline } - if (targetName) { + if (supportsParameter(scenario, 'objective_target') && targetName) { request.target_name = targetName } if (scenario.uses_default_adversarial_target && adversarialTargetName) { @@ -338,8 +346,8 @@ function buildEstimateRequest({ if (maxDatasetSizeValue !== undefined) { request.max_dataset_size = maxDatasetSizeValue } - const harmCategories = parseDatasetNames(harmCategoriesFilter) - const dataTypes = parseDatasetNames(dataTypesFilter) + const harmCategories = supportsDatasets ? parseDatasetNames(harmCategoriesFilter) : [] + const dataTypes = supportsDatasets ? parseDatasetNames(dataTypesFilter) : [] if (harmCategories.length > 0 || dataTypes.length > 0) { request.dataset_filters = { ...(harmCategories.length > 0 ? { harm_categories: harmCategories } : {}), @@ -353,7 +361,7 @@ function buildEstimateRequest({ } function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { - if (!input.targetName) { + if (supportsParameter(input.scenario, 'objective_target') && !input.targetName) { return { ok: false, error: 'Select a target.' } } const estimateResult = buildEstimateRequest(input) @@ -361,9 +369,11 @@ function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { return estimateResult } if ( - !Number.isInteger(input.maxConcurrency) - || input.maxConcurrency < MIN_MAX_CONCURRENCY - || input.maxConcurrency > MAX_MAX_CONCURRENCY + supportsParameter(input.scenario, 'max_concurrency') && ( + !Number.isInteger(input.maxConcurrency) + || input.maxConcurrency < MIN_MAX_CONCURRENCY + || input.maxConcurrency > MAX_MAX_CONCURRENCY + ) ) { return { ok: false, @@ -371,9 +381,11 @@ function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { } } if ( - !Number.isInteger(input.maxRetries) - || input.maxRetries < MIN_MAX_RETRIES - || input.maxRetries > MAX_MAX_RETRIES + supportsParameter(input.scenario, 'max_retries') && ( + !Number.isInteger(input.maxRetries) + || input.maxRetries < MIN_MAX_RETRIES + || input.maxRetries > MAX_MAX_RETRIES + ) ) { return { ok: false, @@ -384,27 +396,16 @@ function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { const estimateRequest = estimateResult.request const request: RunScenarioRequest = { scenario_name: input.scenario.scenario_name, - target_name: input.targetName, - techniques: estimateRequest.techniques, - max_concurrency: input.maxConcurrency, - max_retries: input.maxRetries, - include_baseline: estimateRequest.include_baseline, - labels: input.labels, + ...estimateRequest, } - if (estimateRequest.adversarial_target_name !== undefined) { - request.adversarial_target_name = estimateRequest.adversarial_target_name + if (supportsParameter(input.scenario, 'max_concurrency')) { + request.max_concurrency = input.maxConcurrency } - if (estimateRequest.dataset_names !== undefined) { - request.dataset_names = estimateRequest.dataset_names + if (supportsParameter(input.scenario, 'max_retries')) { + request.max_retries = input.maxRetries } - if (estimateRequest.max_dataset_size !== undefined) { - request.max_dataset_size = estimateRequest.max_dataset_size - } - if (estimateRequest.dataset_filters !== undefined) { - request.dataset_filters = estimateRequest.dataset_filters - } - if (estimateRequest.scenario_params !== undefined) { - request.scenario_params = estimateRequest.scenario_params + if (supportsParameter(input.scenario, 'memory_labels')) { + request.labels = input.labels } return { ok: true, request } } @@ -456,6 +457,12 @@ function ScenarioDetailContent({ .getScenario(decodedScenarioName) .then((data) => { if (cancelled) return + if (!Array.isArray(data.supported_parameter_names)) { + setScenario(null) + setScenarioStatus('error') + setScenarioError('The scenario catalog has no input support metadata. Update the backend and reload.') + return + } setScenario(data) setScenarioStatus('success') setScenarioError(null) @@ -582,10 +589,17 @@ function ScenarioLaunchForm({ ) const dynamicParameters = useMemo( () => scenario.supported_parameters.filter( - (parameter) => !COMMON_SCENARIO_PARAMETER_NAMES.has(parameter.name), + (parameter) => supportsParameter(scenario, parameter.name) + && !COMMON_SCENARIO_PARAMETER_NAMES.has(parameter.name), ), - [scenario.supported_parameters], + [scenario], ) + const supportsObjectiveTarget = supportsParameter(scenario, 'objective_target') + const supportsTechniques = supportsParameter(scenario, 'scenario_techniques') + const supportsBaseline = supportsParameter(scenario, 'include_baseline') + const supportsDatasets = supportsParameter(scenario, 'dataset_config') + const supportsConcurrency = supportsParameter(scenario, 'max_concurrency') + const supportsRetries = supportsParameter(scenario, 'max_retries') const isBaselineForbidden = scenario.baseline_policy === 'forbidden' const [targetName, setTargetName] = useState(() => { @@ -607,8 +621,10 @@ function ScenarioLaunchForm({ }) const [selectedTechniques, setSelectedTechniques] = useState(() => defaultTechniques) const unavailableSelection = ( - targetName !== '' && !targets.some((target) => target.target_registry_name === targetName) - ) || selectedTechniques.some((name) => !techniqueOptions.some((technique) => technique.name === name)) + supportsObjectiveTarget && targetName !== '' + && !targets.some((target) => target.target_registry_name === targetName) + ) || (supportsTechniques + && selectedTechniques.some((name) => !techniqueOptions.some((technique) => technique.name === name))) const [baselineChecked, setBaselineChecked] = useState( () => !isBaselineForbidden && scenario.include_baseline_by_default, ) @@ -656,18 +672,18 @@ function ScenarioLaunchForm({ const selectableTechniques = useMemo( () => [ - { + ...(supportsBaseline ? [{ ...BASELINE_TECHNIQUE, isBaseline: true, disabled: isBaselineForbidden, - }, - ...techniqueOptions.map((technique) => ({ + }] : []), + ...(supportsTechniques ? techniqueOptions.map((technique) => ({ ...technique, isBaseline: false, disabled: false, - })), + })) : []), ], - [isBaselineForbidden, techniqueOptions], + [isBaselineForbidden, supportsBaseline, supportsTechniques, techniqueOptions], ) const techniques = selectedTechniques const maxDatasetSizeOverride = maxDatasetSize.trim() @@ -931,10 +947,11 @@ function ScenarioLaunchForm({ setPreviewOpen(true) } - const techniqueSelectionInvalid = selectedTechniques.length === 0 + const techniqueSelectionInvalid = supportsTechniques && selectedTechniques.length === 0 const displayedEstimateNotes = estimateNotes(estimateState) const displayedEstimate = estimateFromState(estimateState) - const selectedTechniqueCount = selectedTechniques.length + (baselineChecked ? 1 : 0) + const selectedTechniqueCount = (supportsTechniques ? selectedTechniques.length : 0) + + (supportsBaseline && baselineChecked && !isBaselineForbidden ? 1 : 0) const previewDatasets = parseDatasetNames(datasetOverride) const effectiveDatasets = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets const previewHarmCategories = parseDatasetNames(harmCategoriesFilter) @@ -965,9 +982,9 @@ function ScenarioLaunchForm({ onSubmit={handleFormSubmit} noValidate > - {validationError && ( + {validationError && !requestResult.ok && ( - {validationError} + {requestResult.error} )} {(staleSelection || (targetName && unavailableSelection)) && ( @@ -1001,6 +1018,7 @@ function ScenarioLaunchForm({ /> + {supportsObjectiveTarget && (
Objective Target @@ -1036,14 +1054,16 @@ function ScenarioLaunchForm({ )}
+ )} + {(supportsTechniques || supportsBaseline) && (
Techniques - + {supportsTechniques && Select individual techniques, or use a tag to select or clear all techniques with that tag. - + } {techniqueSelectionInvalid && ( Select at least one attack technique. @@ -1101,7 +1121,10 @@ function ScenarioLaunchForm({ })}
+ )} + {(dynamicParameters.length > 0 || scenario.uses_default_adversarial_target + || supportsDatasets || supportsConcurrency || supportsRetries) && (
Parameters @@ -1130,6 +1153,8 @@ function ScenarioLaunchForm({ disabled={submitting} /> )} + {supportsDatasets && ( + <> - + + )} + {supportsConcurrency && setMaxConcurrency(resolveSpinButtonValue(data, maxConcurrency))} data-testid="max-concurrency-input" /> - - + } + {supportsRetries && setMaxRetries(resolveSpinButtonValue(data, maxRetries))} data-testid="max-retries-input" /> - + }
+ )}
Total atomic attacks
{formatAtomicAttackCount(estimateState)}
-
+ {supportsDatasets &&
Dataset size
{datasetSizeLabel}
-
-
+
} + {(supportsTechniques || supportsBaseline) &&
Number techniques
{selectedTechniqueCount}
-
+
} {dynamicParameters.map((parameter) => (
{parameter.name}
@@ -1316,8 +1344,10 @@ function ScenarioLaunchForm({
-
Objective Target
-
{targetName}
+ {supportsObjectiveTarget && <> +
Objective Target
+
{targetName}
+ } {scenario.uses_default_adversarial_target && ( <>
Adversarial Target
@@ -1325,17 +1355,19 @@ function ScenarioLaunchForm({ )}
-
+ {(supportsTechniques || supportsBaseline) &&
Techniques
- {baselineChecked && baseline} - {selectedTechniques.map((name) => ( + {supportsBaseline && baselineChecked && !isBaselineForbidden + && baseline} + {supportsTechniques && selectedTechniques.map((name) => ( {name} ))}
-
+
} + {supportsDatasets && <>
Datasets
@@ -1375,6 +1407,7 @@ function ScenarioLaunchForm({ )}
+ }
Parameters
diff --git a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx index a1491a31bd..1cfc423f32 100644 --- a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx @@ -66,6 +66,10 @@ const SCENARIO: RegisteredScenario = { include_baseline_by_default: true, uses_default_adversarial_target: true, supported_parameters: [], + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', + ], default_run_size: { dataset_size: { kind: 'bounded', value: 1 }, dataset_limit: { state: 'scenario_default' }, diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts index 7f5543c0a1..dd1234862f 100644 --- a/frontend/src/services/api.test.ts +++ b/frontend/src/services/api.test.ts @@ -771,6 +771,7 @@ describe("api service", () => { baseline_policy: "enabled", include_baseline_by_default: true, supported_parameters: [], + supported_parameter_names: [], }, }; (apiClient.get as jest.Mock).mockResolvedValueOnce(mockResponse); diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index b120d240f2..0c7920f127 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -834,6 +834,8 @@ export interface RegisteredScenario { include_baseline_by_default: boolean uses_default_adversarial_target: boolean supported_parameters: Parameter[] + /** All declared names, including opaque inputs with dedicated controls. */ + supported_parameter_names: string[] default_run_size: ScenarioRunSizeEstimateResponse } @@ -850,7 +852,7 @@ export interface ListRegisteredScenariosResponse { export interface RunScenarioRequest { scenario_name: string - target_name: string + target_name?: string | null adversarial_target_name?: string | null initializers?: string[] | null techniques?: string[] | null diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index 89e151be6f..69c7705b7a 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -322,10 +322,8 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque if isinstance(raw_request, dict): # Launch records from before per-run target selection used the server default. raw_request = {"adversarial_target_name": None, **raw_request} - if ( - not isinstance(raw_request, dict) - or any(name not in raw_request for name in _LAUNCH_REQUEST_FIELDS if name != "max_dataset_size") - or raw_request["include_baseline"] is None + if not isinstance(raw_request, dict) or any( + name not in raw_request for name in _LAUNCH_REQUEST_FIELDS if name != "max_dataset_size" ): raise ScenarioRunConflictError("The saved launch configuration is incomplete; resume was not started.") try: @@ -336,12 +334,24 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque raise ScenarioRunConflictError( "The saved launch configuration is invalid; resume was not started." ) from exc - if not request.scenario_name.strip() or not request.target_name.strip(): + if not request.scenario_name.strip() or (request.target_name is not None and not request.target_name.strip()): raise ScenarioRunConflictError("The saved scenario or target registration name is empty.") - identifier = stored.scenario_identifier - if (request.techniques is None and identifier.techniques is None) or ( - request.dataset_names is None and identifier.datasets is None + scenario_class = self._configuration_resolver.resolve_scenario_class(scenario_name=request.scenario_name) + declared_names = {parameter.name for parameter in scenario_class.supported_parameters()} + if any( + parameter_name in declared_names and getattr(request, field_name) is None + for parameter_name, field_name in ( + ("objective_target", "target_name"), + ("include_baseline", "include_baseline"), + ("max_concurrency", "max_concurrency"), + ("max_retries", "max_retries"), + ) ): + raise ScenarioRunConflictError("The saved launch configuration is incomplete; resume was not started.") + identifier = stored.scenario_identifier + if ( + "scenario_techniques" in declared_names and request.techniques is None and identifier.techniques is None + ) or ("dataset_config" in declared_names and request.dataset_names is None and identifier.datasets is None): raise ScenarioRunConflictError("The saved scenario identity is missing techniques or datasets.") custom_params = { name: value @@ -354,9 +364,13 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque "initializers": None, "initializer_args": None, "scenario_params": custom_params, - "techniques": request.techniques if request.techniques is not None else identifier.techniques, - "dataset_names": request.dataset_names if request.dataset_names is not None else identifier.datasets, - "labels": dict(stored.labels), + "techniques": request.techniques + if request.techniques is not None or "scenario_techniques" not in declared_names + else identifier.techniques, + "dataset_names": request.dataset_names + if request.dataset_names is not None or "dataset_config" not in declared_names + else identifier.datasets, + "labels": dict(stored.labels) if "memory_labels" in declared_names else None, }, deep=True, ) @@ -633,7 +647,14 @@ async def _prepare_run_async(self, *, request: RunScenarioRequest) -> _PreparedR owner=request.scenario_name, ) await self._run_initializers_async(request=request) - objective_target = self._configuration_resolver.resolve_target(target_name=request.target_name) + declared_names = {parameter.name for parameter in scenario_class.supported_parameters()} + if "objective_target" in declared_names and not request.target_name: + raise ValueError(f"Scenario '{request.scenario_name}' requires an objective target.") + objective_target = ( + self._configuration_resolver.resolve_target(target_name=request.target_name) + if request.target_name is not None + else None + ) adversarial_target = self._configuration_resolver.resolve_adversarial_target( target_name=request.adversarial_target_name ) @@ -647,8 +668,12 @@ async def _prepare_run_async(self, *, request: RunScenarioRequest) -> _PreparedR max_dataset_size=request.max_dataset_size, dataset_filters=request.dataset_filters, include_baseline=request.include_baseline, - max_concurrency=request.max_concurrency, - max_retries=request.max_retries, + max_concurrency=request.max_concurrency + if "max_concurrency" in declared_names or "max_concurrency" in request.model_fields_set + else None, + max_retries=request.max_retries + if "max_retries" in declared_names or "max_retries" in request.model_fields_set + else None, memory_labels=request.labels, ) scenario = await self._initialize_scenario_async(request=request, init_kwargs=init_kwargs) @@ -1231,9 +1256,13 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_ The fully initialized Scenario instance ready for run_async. """ scenario_registry = ScenarioRegistry.get_registry_singleton() + scenario_class = scenario_registry.get_class(request.scenario_name) + declared_names = {parameter.name for parameter in scenario_class.supported_parameters()} launch_request = {name: getattr(request, name) for name in _LAUNCH_REQUEST_FIELDS} - if launch_request["include_baseline"] is None: - scenario_class = scenario_registry.get_class(request.scenario_name) + for name in ("max_concurrency", "max_retries"): + if name not in declared_names and name not in request.model_fields_set: + launch_request[name] = None + if launch_request["include_baseline"] is None and "include_baseline" in declared_names: baseline_parameter = next( ( parameter diff --git a/pyrit/backend/services/scenario_service.py b/pyrit/backend/services/scenario_service.py index 00cef9743c..ca1c54b0d8 100644 --- a/pyrit/backend/services/scenario_service.py +++ b/pyrit/backend/services/scenario_service.py @@ -73,6 +73,7 @@ def _metadata_to_registered_scenario( for parameter in metadata.supported_parameters if parameter.is_external_input ], + supported_parameter_names=[parameter.name for parameter in metadata.supported_parameters], baseline_policy=metadata.baseline_policy, include_baseline_by_default=metadata.include_baseline_by_default, uses_default_adversarial_target=metadata.uses_default_adversarial_target, diff --git a/pyrit/models/catalog/scenario.py b/pyrit/models/catalog/scenario.py index 9a9f269a25..0314768ce5 100644 --- a/pyrit/models/catalog/scenario.py +++ b/pyrit/models/catalog/scenario.py @@ -373,7 +373,12 @@ class RegisteredScenario(BaseModel): False, description="Whether any available technique uses the shared adversarial target" ) supported_parameters: list[Parameter] = Field( - default_factory=list, description="Scenario-declared custom parameters" + default_factory=list, description="Scenario-declared external input descriptors" + ) + supported_parameter_names: list[str] = Field( + ..., + description="All scenario-declared input names, including opaque inputs handled by dedicated API fields. " + "Clients use these names for input support, not defaults or run-size estimates.", ) default_run_size: ScenarioRunSizeEstimate = Field( default_factory=ScenarioRunSizeEstimate.unavailable, @@ -434,7 +439,9 @@ class RunScenarioRequest(BaseModel): """Request body for starting a scenario run.""" scenario_name: _RequestName = Field(..., description="Scenario name (e.g., 'foundry.red_team_agent')") - target_name: _RequestName = Field(..., description="Name of a registered target from the TargetRegistry") + target_name: _RequestName | None = Field( + None, description="Registered objective target name; required when the scenario declares objective_target" + ) adversarial_target_name: _RequestName | None = Field( None, min_length=1, @@ -461,8 +468,8 @@ class RunScenarioRequest(BaseModel): "Dataset seed filters keyed by field, applied before sampling. Accepted keys: harm_categories, data_types." ), ) - max_concurrency: int = Field(10, ge=1, le=100, description="Maximum concurrent operations") - max_retries: int = Field(0, ge=0, le=20, description="Maximum retry attempts on failure") + max_concurrency: int | None = Field(10, ge=1, le=100, description="Maximum concurrent operations") + max_retries: int | None = Field(0, ge=0, le=20, description="Maximum retry attempts on failure") include_baseline: bool | None = Field( None, description="Override the scenario baseline default; forbidden scenarios reject true" ) diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index a3d04ae1e0..ca681c47b5 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -162,6 +162,75 @@ def _make_request( ) +@pytest.mark.parametrize("supports_common_inputs", [True, False]) +async def test_prepare_and_restore_preserve_common_input_support_async( + patch_central_database: MagicMock, supports_common_inputs: bool +) -> None: + declared = Scam.supported_parameters() if supports_common_inputs else [] + registry = MagicMock(spec=ScenarioRegistry) + registry.get_class.return_value = Scam + registry.create_and_initialize_async = AsyncMock(return_value=MagicMock(spec=Scenario)) + service = ScenarioRunService() + request = RunScenarioRequest( + scenario_name="test.support", + **({"target_name": "my_target"} if supports_common_inputs else {}), + ) + with ( + patch.object(Scam, "supported_parameters", return_value=declared), + patch.object(ScenarioRegistry, "get_registry_singleton", return_value=registry), + patch.object(service, "_run_initializers_async", new_callable=AsyncMock), + patch.object(service._configuration_resolver, "resolve_target") as resolve_target, + ): + await service._prepare_run_async(request=request) + kwargs = registry.create_and_initialize_async.call_args.kwargs + saved = kwargs["initial_metadata"][_svc_mod._LAUNCH_REQUEST_METADATA_KEY] + stored = make_scenario_result( + scenario_name="test.support", + techniques=["prompt_sending"], + datasets=[], + attack_results={}, + metadata={_svc_mod._LAUNCH_REQUEST_METADATA_KEY: saved}, + ) + restored = service._restore_launch_request(stored=stored) + if supports_common_inputs: + resolve_target.assert_called_once() + assert kwargs["max_concurrency"] == restored.max_concurrency == 10 + assert kwargs["max_retries"] == restored.max_retries == 0 + assert restored.target_name == "my_target" + assert restored.include_baseline is True + else: + resolve_target.assert_not_called() + for name in ("objective_target", "max_concurrency", "max_retries", "include_baseline"): + assert name not in kwargs + assert restored.target_name is None + assert restored.max_concurrency is None + assert restored.max_retries is None + assert restored.include_baseline is None + assert restored.techniques is None + assert restored.dataset_names is None + assert restored.labels is None + registry.create_and_initialize_async.reset_mock() + await service._prepare_run_async(request=restored) + for name in ("objective_target", "max_concurrency", "max_retries", "include_baseline"): + assert name not in registry.create_and_initialize_async.call_args.kwargs + await service.close_async() + + +async def test_prepare_requires_target_for_scenarios_that_declare_it_async( + patch_central_database: MagicMock, +) -> None: + registry = MagicMock(spec=ScenarioRegistry) + registry.get_class.return_value = Scam + service = ScenarioRunService() + with ( + patch.object(ScenarioRegistry, "get_registry_singleton", return_value=registry), + patch.object(service, "_run_initializers_async", new_callable=AsyncMock), + pytest.raises(ValueError, match="requires an objective target"), + ): + await service._prepare_run_async(request=RunScenarioRequest(scenario_name="test.support")) + await service.close_async() + + @pytest.mark.parametrize( ("scenario_params", "rejected"), [ diff --git a/tests/unit/backend/test_scenario_service.py b/tests/unit/backend/test_scenario_service.py index 19dbdb2883..df78a77d48 100644 --- a/tests/unit/backend/test_scenario_service.py +++ b/tests/unit/backend/test_scenario_service.py @@ -47,6 +47,8 @@ override_default_adversarial_target, ) from pyrit.scenario.scenarios.airt.scam import Scam +from pyrit.scenario.scenarios.garak.exploitation import Exploitation +from pyrit.score import SubStringScorer from unit.mocks import MockPromptTarget if TYPE_CHECKING: @@ -147,6 +149,27 @@ def test_catalog_lists_only_external_scenario_parameters() -> None: summary = _metadata_to_registered_scenario(metadata=metadata) assert [parameter.name for parameter in summary.supported_parameters] == ["max_concurrency"] + assert summary.supported_parameter_names == ["dataset_config", "max_concurrency"] + + +@pytest.mark.usefixtures("patch_central_database") +@pytest.mark.parametrize("scenario_class", [Scam, Exploitation]) +def test_catalog_support_names_match_real_scenario_declarations(scenario_class: type[Scenario]) -> None: + with ( + patch.object(ScenarioRegistry, "_discover"), + patch.object(Scenario, "_get_default_objective_scorer", return_value=SubStringScorer(substring="test")), + override_default_adversarial_target(MockPromptTarget()), + ): + registry = ScenarioRegistry() + registry.register_class(scenario_class, name="test.support") + metadata = registry.get_registered_class_metadata("test.support") + assert metadata is not None + summary = _metadata_to_registered_scenario(metadata=metadata) + assert summary.supported_parameter_names == [parameter.name for parameter in scenario_class.supported_parameters()] + assert ("dataset_config" in summary.supported_parameter_names) is (scenario_class is Scam) + assert "dataset_config" not in [parameter.name for parameter in summary.supported_parameters] + assert "scenario_techniques" in summary.supported_parameter_names + assert "memory_labels" in summary.supported_parameter_names def test_catalog_describes_scenario_parameters_in_the_form_callers_send() -> None: @@ -1749,6 +1772,7 @@ def test_list_scenarios_with_items(self, client: TestClient) -> None: """Test that GET /api/scenarios/catalog returns scenario data.""" summary = RegisteredScenario( scenario_name="foundry.red_team_agent", + supported_parameter_names=[], scenario_type="RedTeamAgentScenario", description="Red team agent testing", description_markdown='', @@ -1853,6 +1877,7 @@ def test_get_scenario_returns_200(self, client: TestClient) -> None: """Test that GET /api/scenarios/catalog/{name} returns 200 when found.""" summary = RegisteredScenario( scenario_name="foundry.red_team_agent", + supported_parameter_names=[], scenario_type="RedTeamAgentScenario", description="Red team agent testing", default_technique="default", @@ -1994,6 +2019,7 @@ def test_get_scenario_with_dotted_name(self, client: TestClient) -> None: """Test that dotted scenario names (e.g., 'foundry.red_team_agent') work in path.""" summary = RegisteredScenario( scenario_name="garak.encoding", + supported_parameter_names=[], scenario_type="EncodingScenario", description="Encoding scenario", default_technique="all", diff --git a/tests/unit/cli/test_api_client.py b/tests/unit/cli/test_api_client.py index 7185fbf16d..38aa8deb32 100644 --- a/tests/unit/cli/test_api_client.py +++ b/tests/unit/cli/test_api_client.py @@ -343,6 +343,7 @@ def _scenario_payload(*, scenario_name: str = "s1") -> dict: "all_techniques": ["single_turn"], "default_datasets": [], "supported_parameters": [], + "supported_parameter_names": [], } diff --git a/tests/unit/cli/test_output.py b/tests/unit/cli/test_output.py index 35dcb7f00f..925820c173 100644 --- a/tests/unit/cli/test_output.py +++ b/tests/unit/cli/test_output.py @@ -43,6 +43,7 @@ def _make_scenario(**overrides) -> RegisteredScenario: "all_techniques": [], "default_datasets": [], "supported_parameters": [], + "supported_parameter_names": [], } defaults.update(overrides) return RegisteredScenario(**defaults) diff --git a/tests/unit/cli/test_pyrit_scan.py b/tests/unit/cli/test_pyrit_scan.py index e9fa072d37..43c4e1c565 100644 --- a/tests/unit/cli/test_pyrit_scan.py +++ b/tests/unit/cli/test_pyrit_scan.py @@ -423,6 +423,7 @@ def _mock_api_client(): all_techniques=[], default_datasets=[], supported_parameters=[], + supported_parameter_names=[], ) client.start_scenario_run_async.return_value = ScenarioRunSummary( scenario_result_id="test-id-123", @@ -1270,6 +1271,7 @@ def test_main_scenario_not_found_lists_available(self, mock_client_class, _mock_ mock_client.list_scenarios_async.return_value = [ RegisteredScenario( scenario_name="alt_a", + supported_parameter_names=[], scenario_type="X", description="", default_technique="", @@ -1279,6 +1281,7 @@ def test_main_scenario_not_found_lists_available(self, mock_client_class, _mock_ ), RegisteredScenario( scenario_name="alt_b", + supported_parameter_names=[], scenario_type="X", description="", default_technique="", @@ -1751,6 +1754,7 @@ def _build_mock_client(supported_params=None, status="COMPLETED"): client.list_scenarios_async.return_value = [ RegisteredScenario( scenario_name="foo", + supported_parameter_names=[], scenario_type="X", description="", default_technique="", @@ -1768,6 +1772,7 @@ def _build_mock_client(supported_params=None, status="COMPLETED"): all_techniques=[], default_datasets=[], supported_parameters=typed_params, + supported_parameter_names=[parameter.name for parameter in typed_params], ) client.start_scenario_run_async.return_value = ScenarioRunSummary( scenario_result_id="rid", diff --git a/tests/unit/cli/test_pyrit_shell.py b/tests/unit/cli/test_pyrit_shell.py index d678eaa8f3..5b204fe1c2 100644 --- a/tests/unit/cli/test_pyrit_shell.py +++ b/tests/unit/cli/test_pyrit_shell.py @@ -61,6 +61,7 @@ def mock_api_client(): all_techniques=[], default_datasets=[], supported_parameters=[], + supported_parameter_names=[], ) client.close_async = AsyncMock() client.__aenter__ = AsyncMock(return_value=client) @@ -75,6 +76,7 @@ def mock_api_client(): all_techniques=kw.get("all_techniques", []), default_datasets=kw.get("default_datasets", []), supported_parameters=kw.get("supported_parameters", []), + supported_parameter_names=[parameter.name for parameter in kw.get("supported_parameters", [])], ) # Suppress unused-import warning for datetime/timezone helpers used by tests. _ = (datetime, timezone) From 837cc31e7f8c38da605927c63f34453905873b74 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sun, 11 Oct 2026 15:14:26 -0700 Subject: [PATCH 2/3] DOC: Remove detailed scenario input explanation Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c198c14a-c4c7-4e5b-b068-1e62287c0d92 --- frontend/README.md | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/frontend/README.md b/frontend/README.md index aaf9bf3372..4164661c99 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -19,19 +19,6 @@ Failed runs offer **Resume run** on the run page and **Resume** in Scanner Histo See the [GUI guide](../doc/gui/0_gui.md#resuming-a-failed-scanner-run) for resume behavior and the saved launch configuration requirement. -## Supported scenario inputs - -Scanner forms show only inputs declared by the selected scenario. Hidden -inputs are not validated or sent in estimate or launch requests, even if an -older draft contains values for them. Dataset overrides, size limits, and -filters are available only when the scenario declares `dataset_config`. - -The catalog's required `supported_parameter_names` field includes all declared -names, including opaque inputs with dedicated controls. `supported_parameters` -contains only typed external input descriptors. Defaults and run-size estimates -do not determine input support. A catalog response without the names field -shows an error and blocks the form; update the backend and reload. - ## Default labels after runtime changes When the runtime configuration changes, CoPyRIT reloads the server's default From 554466e1ec80713a27ef63ffd0d56206403c67f3 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Sun, 11 Oct 2026 15:46:50 -0700 Subject: [PATCH 3/3] FIX: Save effective scenario concurrency and retry values Resolve supported numeric launch fields using the same parameter declarations and argument precedence as initialization. Keep unsupported fields omitted and cover real SQLite failure/resume with nulls, overrides, and changed defaults. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: c198c14a-c4c7-4e5b-b068-1e62287c0d92 --- .../backend/services/scenario_run_service.py | 24 +++-- tests/unit/backend/test_scenario_resume.py | 94 +++++++++++++++++++ 2 files changed, 111 insertions(+), 7 deletions(-) diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index fecf44d855..f5e24fd967 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -1193,6 +1193,8 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_ ``ScenarioRegistry.create_and_initialize_async`` so the registry owns scenario creation and initialization. The run-specific common parameters are resolved before this method and forwarded as ``init_kwargs``. + The saved launch record retains effective numeric values so resume does + not depend on current parameter defaults. Args: request: The run request (for scenario_name, scenario_params, and @@ -1205,18 +1207,26 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_ """ scenario_registry = ScenarioRegistry.get_registry_singleton() scenario_class = scenario_registry.get_class(request.scenario_name) - declared_names = {parameter.name for parameter in scenario_class.supported_parameters()} + declared = scenario_class.supported_parameters() + declared_names = {parameter.name for parameter in declared} launch_request = {name: getattr(request, name) for name in _LAUNCH_REQUEST_FIELDS} + numeric_parameters = [ + parameter for parameter in declared if parameter.name in {"max_concurrency", "max_retries"} + ] + merged_args = {**(request.scenario_params or {}), **init_kwargs} + resolved_numeric = resolve_declared_params( + declared=numeric_parameters, + raw_args={parameter.name: merged_args.get(parameter.name) for parameter in numeric_parameters}, + owner=f"Scenario '{request.scenario_name}'", + ) for name in ("max_concurrency", "max_retries"): - if name not in declared_names and name not in request.model_fields_set: + if name in declared_names: + launch_request[name] = resolved_numeric[name] + elif name not in request.model_fields_set: launch_request[name] = None if launch_request["include_baseline"] is None and "include_baseline" in declared_names: baseline_parameter = next( - ( - parameter - for parameter in scenario_class.supported_parameters() - if parameter.name == "include_baseline" - ), + (parameter for parameter in declared if parameter.name == "include_baseline"), None, ) baseline = ( diff --git a/tests/unit/backend/test_scenario_resume.py b/tests/unit/backend/test_scenario_resume.py index e9b801a32b..95e492d214 100644 --- a/tests/unit/backend/test_scenario_resume.py +++ b/tests/unit/backend/test_scenario_resume.py @@ -405,6 +405,100 @@ async def fail_second_async(*, normalized_conversation: list[Message]) -> list[M return result +@pytest.mark.parametrize("field", ["max_concurrency", "max_retries"]) +@pytest.mark.parametrize("input_kind", ["omitted", "null", "explicit", "scenario_param", "top_level_wins"]) +async def test_numeric_launch_values_survive_failure_and_default_changes_async( + *, + resume_environment: tuple[ScenarioRunService, MockPromptTarget], + sqlite_instance: SQLiteMemory, + field: str, + input_kind: str, +) -> None: + service, target = resume_environment + scenario_default = 4 if field == "max_concurrency" else 0 + api_default = 10 if field == "max_concurrency" else 0 + explicit_value = 2 if field == "max_concurrency" else 1 + request_values: dict[str, Any] = {} + expected = api_default + if input_kind == "null": + request_values[field] = None + expected = scenario_default + elif input_kind == "explicit": + request_values[field] = explicit_value + expected = explicit_value + elif input_kind == "scenario_param": + request_values.update({field: None, "scenario_params": {field: explicit_value}}) + expected = explicit_value + elif input_kind == "top_level_wins": + request_values.update({field: explicit_value, "scenario_params": {field: explicit_value + 1}}) + expected = explicit_value + + prepared = await service._prepare_run_async( + request=RunScenarioRequest(scenario_name=_SCENARIO_NAME, target_name=_TARGET_NAME, **request_values) + ) + assert getattr(prepared.scenario, f"_{field}") == expected + original_send = target._send_prompt_to_target_async + + async def fail_second_async(*, normalized_conversation: list[Message]) -> list[Message]: + if normalized_conversation[-1].get_value() == _SECOND_OBJECTIVE: + raise RuntimeError("Synthetic target failure") + return await original_send(normalized_conversation=normalized_conversation) + + with patch.object(target, "_send_prompt_to_target_async", side_effect=fail_second_async): + with pytest.raises(ScenarioPartialFailureException): + await prepared.scenario.run_async() + run_id = prepared.scenario._scenario_result_id + assert run_id is not None + stored = await sqlite_instance.get_scenario_result_header_async(scenario_result_id=run_id) + assert stored is not None and stored.scenario_run_state == ScenarioRunState.FAILED + assert stored.metadata[_LAUNCH_REQUEST_METADATA_KEY][field] == expected + + changed_defaults = [ + parameter.model_copy(update={"default": expected + 3}) if parameter.name == field else parameter + for parameter in _OfflineResumeScenario._common_scenario_parameters() + ] + original_run = _OfflineResumeScenario.run_async + + async def run_with_saved_value_async(self: _OfflineResumeScenario) -> None: + assert getattr(self, f"_{field}") == expected + await original_run(self) + + with ( + patch.object(_OfflineResumeScenario, "_common_scenario_parameters", return_value=changed_defaults), + patch.object(_OfflineResumeScenario, "run_async", run_with_saved_value_async), + ): + response = await service.resume_run_async(scenario_result_id=run_id) + await _wait_for_idle_async(service) + assert response.scenario_result_id == run_id + resumed = await sqlite_instance.get_scenario_result_header_async(scenario_result_id=run_id) + assert resumed is not None and resumed.scenario_run_state == ScenarioRunState.COMPLETED + assert resumed.metadata[_LAUNCH_REQUEST_METADATA_KEY][field] == expected + + +@pytest.mark.parametrize("field", ["max_concurrency", "max_retries"]) +async def test_unsupported_numeric_launch_values_remain_omitted_on_resume_async( + *, + resume_environment: tuple[ScenarioRunService, MockPromptTarget], + sqlite_instance: SQLiteMemory, + field: str, +) -> None: + service, _ = resume_environment + declared = [parameter for parameter in _OfflineResumeScenario.supported_parameters() if parameter.name != field] + with patch.object(_OfflineResumeScenario, "supported_parameters", return_value=declared): + prepared = await service._prepare_run_async( + request=RunScenarioRequest(scenario_name=_SCENARIO_NAME, target_name=_TARGET_NAME) + ) + run_id = prepared.scenario._scenario_result_id + assert run_id is not None + stored = await sqlite_instance.get_scenario_result_header_async(scenario_result_id=run_id) + assert stored is not None + assert stored.metadata[_LAUNCH_REQUEST_METADATA_KEY][field] is None + restored = service._restore_launch_request(stored=stored) + assert getattr(restored, field) is None + resumed = await service._prepare_run_async(request=restored) + assert field not in resumed.scenario.params + + async def test_preparation_and_execution_share_sqlite_loop_resources_async( resume_environment: tuple[ScenarioRunService, MockPromptTarget], sqlite_instance: SQLiteMemory ) -> None: