diff --git a/frontend/e2e/numeric-controls.spec.ts b/frontend/e2e/numeric-controls.spec.ts index 09f688297c..a9b7cc8705 100644 --- a/frontend/e2e/numeric-controls.spec.ts +++ b/frontend/e2e/numeric-controls.spec.ts @@ -25,6 +25,10 @@ const SCENARIO: RegisteredScenario = { { name: 'iterations', type_name: 'int', required: false, default: '2', choices: null, is_list: false }, { name: 'temperature', type_name: 'float', required: false, default: '0.5', choices: null, is_list: false }, ], + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', 'iterations', 'temperature', + ], default_run_size: { dataset_size: { kind: 'indeterminate', reason: 'configuration_unavailable', detail: 'Population configuration is not available.' }, dataset_limit: { state: 'scenario_default' }, diff --git a/frontend/e2e/onboarding-tour.spec.ts b/frontend/e2e/onboarding-tour.spec.ts index 3572c48193..dae177aaa3 100644 --- a/frontend/e2e/onboarding-tour.spec.ts +++ b/frontend/e2e/onboarding-tour.spec.ts @@ -22,6 +22,10 @@ const TOUR_SCENARIO: RegisteredScenario = { baseline_policy: "disabled", include_baseline_by_default: false, supported_parameters: [], + supported_parameter_names: [ + "objective_target", "scenario_techniques", "dataset_config", "memory_labels", + "max_concurrency", "max_retries", "include_baseline", + ], default_run_size: { status: "unavailable", dataset_size: { kind: "indeterminate", detail: "Mock estimate only." }, diff --git a/frontend/e2e/scenario-history.spec.ts b/frontend/e2e/scenario-history.spec.ts index 23f2e4c9a1..1abb526004 100644 --- a/frontend/e2e/scenario-history.spec.ts +++ b/frontend/e2e/scenario-history.spec.ts @@ -91,6 +91,11 @@ const catalogScenario = { baseline_policy: "enabled", include_baseline_by_default: false, uses_default_adversarial_target: false, + supported_parameter_names: [ + "objective_target", "scenario_techniques", "dataset_config", "memory_labels", + "max_concurrency", "max_retries", "include_baseline", "num_jailbreaks", + "num_jailbreak_attempts", "jailbreak_names", + ], supported_parameters: [ { name: "num_jailbreaks", diff --git a/frontend/src/App.labels.test.tsx b/frontend/src/App.labels.test.tsx index c2ad8fa90e..9f17d1610f 100644 --- a/frontend/src/App.labels.test.tsx +++ b/frontend/src/App.labels.test.tsx @@ -77,6 +77,10 @@ const SCENARIO: RegisteredScenario = { baseline_policy: 'forbidden', include_baseline_by_default: false, supported_parameters: [], + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', + ], uses_default_adversarial_target: false, default_run_size: { dataset_size: { kind: 'bounded', value: 1 }, diff --git a/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx b/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx index 84e19457c1..7bbb1b639f 100644 --- a/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioCatalog.test.tsx @@ -66,6 +66,7 @@ function makeScenario(overrides: Partial & { scenario_name: include_baseline_by_default: true, uses_default_adversarial_target: true, supported_parameters: [], + supported_parameter_names: [], default_run_size: { dataset_size: { kind: 'indeterminate', detail: 'Population configuration is not available.' }, dataset_limit: { state: 'scenario_default' }, diff --git a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx index 477c78c8fb..c3ab93e110 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.test.tsx @@ -4,6 +4,7 @@ import { FluentProvider, webLightTheme } from '@fluentui/react-components' import { MemoryRouter, Route, Routes } from 'react-router' import { scenariosApi, targetsApi } from '@/services/api' +import { useRuntime } from '@/hooks/useRuntime' import type { RegisteredScenario, ScenarioRunSizeEstimateResponse, @@ -12,6 +13,8 @@ import type { import ScenarioDetail from './ScenarioDetail' +jest.mock('@/hooks/useRuntime', () => ({ useRuntime: jest.fn() })) + jest.mock('@/services/api', () => ({ scenariosApi: { estimateRun: jest.fn(), @@ -27,6 +30,7 @@ const mockGetScenario = scenariosApi.getScenario as jest.Mock const mockEstimateRun = scenariosApi.estimateRun as jest.Mock const mockStartRun = scenariosApi.startRun as jest.Mock const mockListTargets = targetsApi.listTargets as jest.Mock +const mockUseRuntime = jest.mocked(useRuntime) const mockNavigate = jest.fn() const RAW_IMAGE_HTML = ['<', 'img src=x onerror="alert(1)">'].join('') @@ -63,6 +67,11 @@ function makeScenario(overrides: Partial = {}): RegisteredSc include_baseline_by_default: true, uses_default_adversarial_target: true, supported_parameters: [], + supported_parameter_names: overrides.supported_parameter_names ?? [ + 'objective_target', 'scenario_techniques', 'technique_converters', 'dataset_config', + 'memory_labels', 'max_concurrency', 'max_retries', 'include_baseline', + ...(overrides.supported_parameters ?? []).map((parameter) => parameter.name), + ], default_run_size: { dataset_size: { kind: 'indeterminate', detail: 'Population configuration is not available.' }, dataset_limit: { state: 'scenario_default' }, @@ -190,6 +199,7 @@ function renderDetail( describe('ScenarioDetail', () => { beforeEach(() => { jest.clearAllMocks() + mockUseRuntime.mockReturnValue({ ready: true, state: 'ready', generation: 'first' }) mockGetScenario.mockReset() mockEstimateRun.mockReset() mockListTargets.mockReset() @@ -203,6 +213,137 @@ describe('ScenarioDetail', () => { jest.useRealTimers() }) + it('blocks a legacy catalog response without support names instead of guessing common support', async () => { + const legacyScenario = makeScenario() + Reflect.deleteProperty(legacyScenario, 'supported_parameter_names') + mockGetScenario.mockResolvedValue(legacyScenario) + + renderDetail('/scanner/foundry.red_team_agent') + + expect(await screen.findByText(/catalog has no input support metadata/i)).toBeInTheDocument() + expect(screen.queryByRole('form')).not.toBeInTheDocument() + expect(mockEstimateRun).not.toHaveBeenCalled() + expect(mockStartRun).not.toHaveBeenCalled() + }) + + it.each([ + ['dataset_config', 'Dataset override', ['dataset_names', 'max_dataset_size', 'dataset_filters']], + ['scenario_techniques', 'default_technique', ['techniques']], + ['include_baseline', 'baseline', ['include_baseline']], + ['max_concurrency', 'Max concurrency', ['max_concurrency']], + ['max_retries', 'Max retries', ['max_retries']], + ['memory_labels', null, ['labels']], + ['objective_target', 'Objective Target', ['target_name']], + ])('omits controls and request aliases for undeclared %s', async ( + name: string, label: string | null, requestFields: string[], + ) => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const scenario = makeScenario() + scenario.supported_parameter_names = scenario.supported_parameter_names.filter( + (parameterName) => parameterName !== name, + ) + // Defaults and estimate data do not establish support for an input. + mockGetScenario.mockResolvedValue(scenario) + mockEstimateRun.mockResolvedValue(makeEstimate(8)) + renderDetail('/scanner/foundry.red_team_agent') + await flushRenderedPromises() + await advanceTimers(300) + + if (label) { + expect(screen.queryByLabelText(label)).not.toBeInTheDocument() + } + if (name === 'dataset_config') { + expect(screen.queryByLabelText('Max dataset size')).not.toBeInTheDocument() + expect(screen.queryByLabelText('Harm categories')).not.toBeInTheDocument() + expect(screen.queryByLabelText('Data types')).not.toBeInTheDocument() + expect(screen.queryByRole('checkbox', { name: 'No total dataset limit' })).not.toBeInTheDocument() + } + expect(mockEstimateRun).toHaveBeenCalledTimes(1) + const preview = await openRunPreview(user) + if (name === 'dataset_config') { + expect(within(preview).queryByText('Datasets')).not.toBeInTheDocument() + expect(within(preview).queryByText('Dataset filters')).not.toBeInTheDocument() + } + await user.click(screen.getByTestId('confirm-launch-scenario-btn')) + expect(mockStartRun).toHaveBeenCalledTimes(1) + for (const field of requestFields) { + expect(mockEstimateRun.mock.calls[0][1]).not.toHaveProperty(field) + expect(mockStartRun.mock.calls[0][0]).not.toHaveProperty(field) + } + }) + + it('keeps baseline selection independent when attack technique selection is unsupported', async () => { + const user = userEvent.setup() + mockGetScenario.mockResolvedValue(makeScenario({ + supported_parameter_names: ['objective_target', 'include_baseline'], + all_techniques: [], + default_techniques: [], + })) + renderDetail('/scanner/foundry.red_team_agent') + const baseline = await screen.findByRole('checkbox', { name: 'baseline' }) + await user.click(baseline) + await confirmRunPreview(user) + + expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: 'foundry.red_team_agent', + target_name: 'target-a', + include_baseline: false, + }) + expect(screen.queryByText('Select at least one attack technique.')).not.toBeInTheDocument() + }) + + it('drops unsupported invalid drafts after catalog refresh and preserves supported values', async () => { + jest.useFakeTimers() + const user = userEvent.setup({ advanceTimers: jest.advanceTimersByTime }) + const parameters = [ + { name: 'iterations', type_name: 'int', required: true, default: null, choices: null, is_list: false }, + { name: 'note', type_name: 'str', required: true, default: null, choices: null, is_list: false }, + ] + mockGetScenario.mockResolvedValue(makeScenario({ supported_parameters: parameters })) + mockEstimateRun.mockResolvedValue(makeEstimate(8)) + const detail = renderDetail('/scanner/foundry.red_team_agent') + await flushRenderedPromises() + await user.type(screen.getByLabelText(/^iterations/), '1.5') + await user.type(screen.getByLabelText(/^note/), 'Keep this value') + await user.type(screen.getByLabelText('Dataset override'), 'stale-dataset') + await user.type(screen.getByLabelText('Max dataset size'), '0') + await user.type(screen.getByLabelText('Harm categories'), 'stale-harm') + await user.type(screen.getByLabelText('Data types'), 'stale-type') + for (const label of ['Max concurrency', 'Max retries']) { + const input = screen.getByRole('spinbutton', { name: label }) + await user.clear(input) + await user.type(input, '500') + await user.tab() + } + await user.click(screen.getByRole('button', { name: 'Launch scan' })) + expect(screen.getByRole('alert')).toHaveTextContent('iterations must be an integer.') + + mockGetScenario.mockResolvedValue(makeScenario({ + supported_parameter_names: ['note'], + // A removed external descriptor must not validate or submit a stale value. + supported_parameters: parameters, + uses_default_adversarial_target: false, + })) + mockUseRuntime.mockReturnValue({ ready: true, state: 'ready', generation: 'second' }) + detail.updateDefaults({ targets: [] }) + await flushRenderedPromises() + await user.click(screen.getByRole('button', { name: 'I have reviewed the refreshed selections' })) + await advanceTimers(300) + + expect(screen.getByLabelText(/^note/)).toHaveValue('Keep this value') + expect(screen.queryByLabelText(/^iterations/)).not.toBeInTheDocument() + expect(screen.queryByRole('alert')).not.toBeInTheDocument() + expect(mockEstimateRun.mock.calls.at(-1)?.[1]).toEqual({ + scenario_params: { note: 'Keep this value' }, + }) + await confirmRunPreview(user) + expect(mockStartRun).toHaveBeenCalledWith({ + scenario_name: 'foundry.red_team_agent', + scenario_params: { note: 'Keep this value' }, + }) + }) + it('shows a loading state while fetching', () => { mockGetScenario.mockReturnValue(new Promise(() => {})) renderDetail('/scanner/foundry.red_team_agent') diff --git a/frontend/src/components/Scenarios/ScenarioDetail.tsx b/frontend/src/components/Scenarios/ScenarioDetail.tsx index f4378c4f3c..eb4cb137ba 100644 --- a/frontend/src/components/Scenarios/ScenarioDetail.tsx +++ b/frontend/src/components/Scenarios/ScenarioDetail.tsx @@ -63,8 +63,8 @@ import { mapScenarioRunEstimate } from './scenarioRunEstimateAdapter' import { techniqueSetName } from './scenarioTechniqueSets' /** - * Common/opaque parameters every scenario declares via - * `Scenario._common_scenario_parameters` — the launch form already exposes a + * Common/opaque parameters the base scenario declares via + * `Scenario._common_scenario_parameters` — the launch form exposes a * purpose-built control for each of these (target, techniques, datasets, * labels, concurrency, retries, baseline), and `technique_converters` has no * UI at all. They're hidden from the dynamic scenario-specific parameter list. @@ -88,6 +88,10 @@ const DEFAULT_MAX_CONCURRENCY = 10 const DEFAULT_MAX_RETRIES = 0 const ESTIMATE_DEBOUNCE_MS = 300 +function supportsParameter(scenario: RegisteredScenario, name: string): boolean { + return scenario.supported_parameter_names.includes(name) +} + function targetOptionLabel(target: TargetInstance): string { const modelName = targetModelName(target) return modelName @@ -297,7 +301,7 @@ function buildEstimateRequest({ dataTypesFilter, includeBaseline, }: BuildEstimateRequestInput): BuildEstimateRequestResult { - if (techniques.length === 0) { + if (supportsParameter(scenario, 'scenario_techniques') && techniques.length === 0) { return { ok: false, error: 'Select at least one technique.' } } @@ -311,7 +315,8 @@ function buildEstimateRequest({ } let maxDatasetSizeValue: number | 'all' | undefined - const trimmedMaxDatasetSize = maxDatasetSize.trim() + const supportsDatasets = supportsParameter(scenario, 'dataset_config') + const trimmedMaxDatasetSize = supportsDatasets ? maxDatasetSize.trim() : '' if (trimmedMaxDatasetSize === 'all') { maxDatasetSizeValue = 'all' } else if (trimmedMaxDatasetSize.length > 0) { @@ -321,12 +326,15 @@ function buildEstimateRequest({ } maxDatasetSizeValue = parsed } - const datasetNames = parseDatasetNames(datasetOverride) - const request: ScenarioRunSizeEstimateRequest = { - techniques, - include_baseline: includeBaseline, + const datasetNames = supportsDatasets ? parseDatasetNames(datasetOverride) : [] + const request: ScenarioRunSizeEstimateRequest = {} + if (supportsParameter(scenario, 'scenario_techniques')) { + request.techniques = techniques + } + if (supportsParameter(scenario, 'include_baseline')) { + request.include_baseline = includeBaseline } - if (targetName) { + if (supportsParameter(scenario, 'objective_target') && targetName) { request.target_name = targetName } if (scenario.uses_default_adversarial_target && adversarialTargetName) { @@ -338,8 +346,8 @@ function buildEstimateRequest({ if (maxDatasetSizeValue !== undefined) { request.max_dataset_size = maxDatasetSizeValue } - const harmCategories = parseDatasetNames(harmCategoriesFilter) - const dataTypes = parseDatasetNames(dataTypesFilter) + const harmCategories = supportsDatasets ? parseDatasetNames(harmCategoriesFilter) : [] + const dataTypes = supportsDatasets ? parseDatasetNames(dataTypesFilter) : [] if (harmCategories.length > 0 || dataTypes.length > 0) { request.dataset_filters = { ...(harmCategories.length > 0 ? { harm_categories: harmCategories } : {}), @@ -353,7 +361,7 @@ function buildEstimateRequest({ } function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { - if (!input.targetName) { + if (supportsParameter(input.scenario, 'objective_target') && !input.targetName) { return { ok: false, error: 'Select a target.' } } const estimateResult = buildEstimateRequest(input) @@ -361,9 +369,11 @@ function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { return estimateResult } if ( - !Number.isInteger(input.maxConcurrency) - || input.maxConcurrency < MIN_MAX_CONCURRENCY - || input.maxConcurrency > MAX_MAX_CONCURRENCY + supportsParameter(input.scenario, 'max_concurrency') && ( + !Number.isInteger(input.maxConcurrency) + || input.maxConcurrency < MIN_MAX_CONCURRENCY + || input.maxConcurrency > MAX_MAX_CONCURRENCY + ) ) { return { ok: false, @@ -371,9 +381,11 @@ function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { } } if ( - !Number.isInteger(input.maxRetries) - || input.maxRetries < MIN_MAX_RETRIES - || input.maxRetries > MAX_MAX_RETRIES + supportsParameter(input.scenario, 'max_retries') && ( + !Number.isInteger(input.maxRetries) + || input.maxRetries < MIN_MAX_RETRIES + || input.maxRetries > MAX_MAX_RETRIES + ) ) { return { ok: false, @@ -384,27 +396,16 @@ function buildRunRequest(input: BuildRunRequestInput): BuildRunRequestResult { const estimateRequest = estimateResult.request const request: RunScenarioRequest = { scenario_name: input.scenario.scenario_name, - target_name: input.targetName, - techniques: estimateRequest.techniques, - max_concurrency: input.maxConcurrency, - max_retries: input.maxRetries, - include_baseline: estimateRequest.include_baseline, - labels: input.labels, + ...estimateRequest, } - if (estimateRequest.adversarial_target_name !== undefined) { - request.adversarial_target_name = estimateRequest.adversarial_target_name + if (supportsParameter(input.scenario, 'max_concurrency')) { + request.max_concurrency = input.maxConcurrency } - if (estimateRequest.dataset_names !== undefined) { - request.dataset_names = estimateRequest.dataset_names + if (supportsParameter(input.scenario, 'max_retries')) { + request.max_retries = input.maxRetries } - if (estimateRequest.max_dataset_size !== undefined) { - request.max_dataset_size = estimateRequest.max_dataset_size - } - if (estimateRequest.dataset_filters !== undefined) { - request.dataset_filters = estimateRequest.dataset_filters - } - if (estimateRequest.scenario_params !== undefined) { - request.scenario_params = estimateRequest.scenario_params + if (supportsParameter(input.scenario, 'memory_labels')) { + request.labels = input.labels } return { ok: true, request } } @@ -456,6 +457,12 @@ function ScenarioDetailContent({ .getScenario(decodedScenarioName) .then((data) => { if (cancelled) return + if (!Array.isArray(data.supported_parameter_names)) { + setScenario(null) + setScenarioStatus('error') + setScenarioError('The scenario catalog has no input support metadata. Update the backend and reload.') + return + } setScenario(data) setScenarioStatus('success') setScenarioError(null) @@ -582,10 +589,17 @@ function ScenarioLaunchForm({ ) const dynamicParameters = useMemo( () => scenario.supported_parameters.filter( - (parameter) => !COMMON_SCENARIO_PARAMETER_NAMES.has(parameter.name), + (parameter) => supportsParameter(scenario, parameter.name) + && !COMMON_SCENARIO_PARAMETER_NAMES.has(parameter.name), ), - [scenario.supported_parameters], + [scenario], ) + const supportsObjectiveTarget = supportsParameter(scenario, 'objective_target') + const supportsTechniques = supportsParameter(scenario, 'scenario_techniques') + const supportsBaseline = supportsParameter(scenario, 'include_baseline') + const supportsDatasets = supportsParameter(scenario, 'dataset_config') + const supportsConcurrency = supportsParameter(scenario, 'max_concurrency') + const supportsRetries = supportsParameter(scenario, 'max_retries') const isBaselineForbidden = scenario.baseline_policy === 'forbidden' const [targetName, setTargetName] = useState(() => { @@ -607,8 +621,10 @@ function ScenarioLaunchForm({ }) const [selectedTechniques, setSelectedTechniques] = useState(() => defaultTechniques) const unavailableSelection = ( - targetName !== '' && !targets.some((target) => target.target_registry_name === targetName) - ) || selectedTechniques.some((name) => !techniqueOptions.some((technique) => technique.name === name)) + supportsObjectiveTarget && targetName !== '' + && !targets.some((target) => target.target_registry_name === targetName) + ) || (supportsTechniques + && selectedTechniques.some((name) => !techniqueOptions.some((technique) => technique.name === name))) const [baselineChecked, setBaselineChecked] = useState( () => !isBaselineForbidden && scenario.include_baseline_by_default, ) @@ -656,18 +672,18 @@ function ScenarioLaunchForm({ const selectableTechniques = useMemo( () => [ - { + ...(supportsBaseline ? [{ ...BASELINE_TECHNIQUE, isBaseline: true, disabled: isBaselineForbidden, - }, - ...techniqueOptions.map((technique) => ({ + }] : []), + ...(supportsTechniques ? techniqueOptions.map((technique) => ({ ...technique, isBaseline: false, disabled: false, - })), + })) : []), ], - [isBaselineForbidden, techniqueOptions], + [isBaselineForbidden, supportsBaseline, supportsTechniques, techniqueOptions], ) const techniques = selectedTechniques const maxDatasetSizeOverride = maxDatasetSize.trim() @@ -931,10 +947,11 @@ function ScenarioLaunchForm({ setPreviewOpen(true) } - const techniqueSelectionInvalid = selectedTechniques.length === 0 + const techniqueSelectionInvalid = supportsTechniques && selectedTechniques.length === 0 const displayedEstimateNotes = estimateNotes(estimateState) const displayedEstimate = estimateFromState(estimateState) - const selectedTechniqueCount = selectedTechniques.length + (baselineChecked ? 1 : 0) + const selectedTechniqueCount = (supportsTechniques ? selectedTechniques.length : 0) + + (supportsBaseline && baselineChecked && !isBaselineForbidden ? 1 : 0) const previewDatasets = parseDatasetNames(datasetOverride) const effectiveDatasets = previewDatasets.length > 0 ? previewDatasets : scenario.default_datasets const previewHarmCategories = parseDatasetNames(harmCategoriesFilter) @@ -965,9 +982,9 @@ function ScenarioLaunchForm({ onSubmit={handleFormSubmit} noValidate > - {validationError && ( + {validationError && !requestResult.ok && ( - {validationError} + {requestResult.error} )} {(staleSelection || (targetName && unavailableSelection)) && ( @@ -1001,6 +1018,7 @@ function ScenarioLaunchForm({ /> + {supportsObjectiveTarget && (
Objective Target @@ -1036,14 +1054,16 @@ function ScenarioLaunchForm({ )}
+ )} + {(supportsTechniques || supportsBaseline) && (
Techniques - + {supportsTechniques && Select individual techniques, or use a tag to select or clear all techniques with that tag. - + } {techniqueSelectionInvalid && ( Select at least one attack technique. @@ -1101,7 +1121,10 @@ function ScenarioLaunchForm({ })}
+ )} + {(dynamicParameters.length > 0 || scenario.uses_default_adversarial_target + || supportsDatasets || supportsConcurrency || supportsRetries) && (
Parameters @@ -1130,6 +1153,8 @@ function ScenarioLaunchForm({ disabled={submitting} /> )} + {supportsDatasets && ( + <> - + + )} + {supportsConcurrency && setMaxConcurrency(resolveSpinButtonValue(data, maxConcurrency))} data-testid="max-concurrency-input" /> - - + } + {supportsRetries && setMaxRetries(resolveSpinButtonValue(data, maxRetries))} data-testid="max-retries-input" /> - + }
+ )}
Total atomic attacks
{formatAtomicAttackCount(estimateState)}
-
+ {supportsDatasets &&
Dataset size
{datasetSizeLabel}
-
-
+
} + {(supportsTechniques || supportsBaseline) &&
Number techniques
{selectedTechniqueCount}
-
+
} {dynamicParameters.map((parameter) => (
{parameter.name}
@@ -1316,8 +1344,10 @@ function ScenarioLaunchForm({
-
Objective Target
-
{targetName}
+ {supportsObjectiveTarget && <> +
Objective Target
+
{targetName}
+ } {scenario.uses_default_adversarial_target && ( <>
Adversarial Target
@@ -1325,17 +1355,19 @@ function ScenarioLaunchForm({ )}
-
+ {(supportsTechniques || supportsBaseline) &&
Techniques
- {baselineChecked && baseline} - {selectedTechniques.map((name) => ( + {supportsBaseline && baselineChecked && !isBaselineForbidden + && baseline} + {supportsTechniques && selectedTechniques.map((name) => ( {name} ))}
-
+
} + {supportsDatasets && <>
Datasets
@@ -1375,6 +1407,7 @@ function ScenarioLaunchForm({ )}
+ }
Parameters
diff --git a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx index a1491a31bd..1cfc423f32 100644 --- a/frontend/src/components/Scenarios/ScenarioFlow.test.tsx +++ b/frontend/src/components/Scenarios/ScenarioFlow.test.tsx @@ -66,6 +66,10 @@ const SCENARIO: RegisteredScenario = { include_baseline_by_default: true, uses_default_adversarial_target: true, supported_parameters: [], + supported_parameter_names: [ + 'objective_target', 'scenario_techniques', 'dataset_config', 'memory_labels', + 'max_concurrency', 'max_retries', 'include_baseline', + ], default_run_size: { dataset_size: { kind: 'bounded', value: 1 }, dataset_limit: { state: 'scenario_default' }, diff --git a/frontend/src/services/api.test.ts b/frontend/src/services/api.test.ts index 7f5543c0a1..dd1234862f 100644 --- a/frontend/src/services/api.test.ts +++ b/frontend/src/services/api.test.ts @@ -771,6 +771,7 @@ describe("api service", () => { baseline_policy: "enabled", include_baseline_by_default: true, supported_parameters: [], + supported_parameter_names: [], }, }; (apiClient.get as jest.Mock).mockResolvedValueOnce(mockResponse); diff --git a/frontend/src/types/index.ts b/frontend/src/types/index.ts index b120d240f2..0c7920f127 100644 --- a/frontend/src/types/index.ts +++ b/frontend/src/types/index.ts @@ -834,6 +834,8 @@ export interface RegisteredScenario { include_baseline_by_default: boolean uses_default_adversarial_target: boolean supported_parameters: Parameter[] + /** All declared names, including opaque inputs with dedicated controls. */ + supported_parameter_names: string[] default_run_size: ScenarioRunSizeEstimateResponse } @@ -850,7 +852,7 @@ export interface ListRegisteredScenariosResponse { export interface RunScenarioRequest { scenario_name: string - target_name: string + target_name?: string | null adversarial_target_name?: string | null initializers?: string[] | null techniques?: string[] | null diff --git a/pyrit/backend/services/scenario_run_service.py b/pyrit/backend/services/scenario_run_service.py index 2d7e38919a..f5e24fd967 100644 --- a/pyrit/backend/services/scenario_run_service.py +++ b/pyrit/backend/services/scenario_run_service.py @@ -324,10 +324,8 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque if isinstance(raw_request, dict): # Launch records from before per-run target selection used the server default. raw_request = {"adversarial_target_name": None, **raw_request} - if ( - not isinstance(raw_request, dict) - or any(name not in raw_request for name in _LAUNCH_REQUEST_FIELDS if name != "max_dataset_size") - or raw_request["include_baseline"] is None + if not isinstance(raw_request, dict) or any( + name not in raw_request for name in _LAUNCH_REQUEST_FIELDS if name != "max_dataset_size" ): raise ScenarioRunConflictError("The saved launch configuration is incomplete; resume was not started.") try: @@ -338,12 +336,24 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque raise ScenarioRunConflictError( "The saved launch configuration is invalid; resume was not started." ) from exc - if not request.scenario_name.strip() or not request.target_name.strip(): + if not request.scenario_name.strip() or (request.target_name is not None and not request.target_name.strip()): raise ScenarioRunConflictError("The saved scenario or target registration name is empty.") - identifier = stored.scenario_identifier - if (request.techniques is None and identifier.techniques is None) or ( - request.dataset_names is None and identifier.datasets is None + scenario_class = self._configuration_resolver.resolve_scenario_class(scenario_name=request.scenario_name) + declared_names = {parameter.name for parameter in scenario_class.supported_parameters()} + if any( + parameter_name in declared_names and getattr(request, field_name) is None + for parameter_name, field_name in ( + ("objective_target", "target_name"), + ("include_baseline", "include_baseline"), + ("max_concurrency", "max_concurrency"), + ("max_retries", "max_retries"), + ) ): + raise ScenarioRunConflictError("The saved launch configuration is incomplete; resume was not started.") + identifier = stored.scenario_identifier + if ( + "scenario_techniques" in declared_names and request.techniques is None and identifier.techniques is None + ) or ("dataset_config" in declared_names and request.dataset_names is None and identifier.datasets is None): raise ScenarioRunConflictError("The saved scenario identity is missing techniques or datasets.") custom_params = { name: value @@ -356,9 +366,13 @@ def _restore_launch_request(self, *, stored: ScenarioResult) -> RunScenarioReque "initializers": None, "initializer_args": None, "scenario_params": custom_params, - "techniques": request.techniques if request.techniques is not None else identifier.techniques, - "dataset_names": request.dataset_names if request.dataset_names is not None else identifier.datasets, - "labels": dict(stored.labels), + "techniques": request.techniques + if request.techniques is not None or "scenario_techniques" not in declared_names + else identifier.techniques, + "dataset_names": request.dataset_names + if request.dataset_names is not None or "dataset_config" not in declared_names + else identifier.datasets, + "labels": dict(stored.labels) if "memory_labels" in declared_names else None, }, deep=True, ) @@ -572,7 +586,14 @@ async def _prepare_run_async(self, *, request: RunScenarioRequest) -> _PreparedR owner=request.scenario_name, ) await self._run_initializers_async(request=request) - objective_target = self._configuration_resolver.resolve_target(target_name=request.target_name) + declared_names = {parameter.name for parameter in scenario_class.supported_parameters()} + if "objective_target" in declared_names and not request.target_name: + raise ValueError(f"Scenario '{request.scenario_name}' requires an objective target.") + objective_target = ( + self._configuration_resolver.resolve_target(target_name=request.target_name) + if request.target_name is not None + else None + ) adversarial_target = self._configuration_resolver.resolve_adversarial_target( target_name=request.adversarial_target_name ) @@ -587,8 +608,12 @@ async def _prepare_run_async(self, *, request: RunScenarioRequest) -> _PreparedR max_dataset_size=request.max_dataset_size, dataset_filters=request.dataset_filters, include_baseline=request.include_baseline, - max_concurrency=request.max_concurrency, - max_retries=request.max_retries, + max_concurrency=request.max_concurrency + if "max_concurrency" in declared_names or "max_concurrency" in request.model_fields_set + else None, + max_retries=request.max_retries + if "max_retries" in declared_names or "max_retries" in request.model_fields_set + else None, memory_labels=request.labels, ) scenario = await self._initialize_scenario_async(request=request, init_kwargs=init_kwargs) @@ -1168,6 +1193,8 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_ ``ScenarioRegistry.create_and_initialize_async`` so the registry owns scenario creation and initialization. The run-specific common parameters are resolved before this method and forwarded as ``init_kwargs``. + The saved launch record retains effective numeric values so resume does + not depend on current parameter defaults. Args: request: The run request (for scenario_name, scenario_params, and @@ -1179,15 +1206,27 @@ async def _initialize_scenario_async(self, *, request: RunScenarioRequest, init_ The fully initialized Scenario instance ready for run_async. """ scenario_registry = ScenarioRegistry.get_registry_singleton() + scenario_class = scenario_registry.get_class(request.scenario_name) + declared = scenario_class.supported_parameters() + declared_names = {parameter.name for parameter in declared} launch_request = {name: getattr(request, name) for name in _LAUNCH_REQUEST_FIELDS} - if launch_request["include_baseline"] is None: - scenario_class = scenario_registry.get_class(request.scenario_name) + numeric_parameters = [ + parameter for parameter in declared if parameter.name in {"max_concurrency", "max_retries"} + ] + merged_args = {**(request.scenario_params or {}), **init_kwargs} + resolved_numeric = resolve_declared_params( + declared=numeric_parameters, + raw_args={parameter.name: merged_args.get(parameter.name) for parameter in numeric_parameters}, + owner=f"Scenario '{request.scenario_name}'", + ) + for name in ("max_concurrency", "max_retries"): + if name in declared_names: + launch_request[name] = resolved_numeric[name] + elif name not in request.model_fields_set: + launch_request[name] = None + if launch_request["include_baseline"] is None and "include_baseline" in declared_names: baseline_parameter = next( - ( - parameter - for parameter in scenario_class.supported_parameters() - if parameter.name == "include_baseline" - ), + (parameter for parameter in declared if parameter.name == "include_baseline"), None, ) baseline = ( diff --git a/pyrit/backend/services/scenario_service.py b/pyrit/backend/services/scenario_service.py index 00cef9743c..ca1c54b0d8 100644 --- a/pyrit/backend/services/scenario_service.py +++ b/pyrit/backend/services/scenario_service.py @@ -73,6 +73,7 @@ def _metadata_to_registered_scenario( for parameter in metadata.supported_parameters if parameter.is_external_input ], + supported_parameter_names=[parameter.name for parameter in metadata.supported_parameters], baseline_policy=metadata.baseline_policy, include_baseline_by_default=metadata.include_baseline_by_default, uses_default_adversarial_target=metadata.uses_default_adversarial_target, diff --git a/pyrit/models/catalog/scenario.py b/pyrit/models/catalog/scenario.py index 9a9f269a25..0314768ce5 100644 --- a/pyrit/models/catalog/scenario.py +++ b/pyrit/models/catalog/scenario.py @@ -373,7 +373,12 @@ class RegisteredScenario(BaseModel): False, description="Whether any available technique uses the shared adversarial target" ) supported_parameters: list[Parameter] = Field( - default_factory=list, description="Scenario-declared custom parameters" + default_factory=list, description="Scenario-declared external input descriptors" + ) + supported_parameter_names: list[str] = Field( + ..., + description="All scenario-declared input names, including opaque inputs handled by dedicated API fields. " + "Clients use these names for input support, not defaults or run-size estimates.", ) default_run_size: ScenarioRunSizeEstimate = Field( default_factory=ScenarioRunSizeEstimate.unavailable, @@ -434,7 +439,9 @@ class RunScenarioRequest(BaseModel): """Request body for starting a scenario run.""" scenario_name: _RequestName = Field(..., description="Scenario name (e.g., 'foundry.red_team_agent')") - target_name: _RequestName = Field(..., description="Name of a registered target from the TargetRegistry") + target_name: _RequestName | None = Field( + None, description="Registered objective target name; required when the scenario declares objective_target" + ) adversarial_target_name: _RequestName | None = Field( None, min_length=1, @@ -461,8 +468,8 @@ class RunScenarioRequest(BaseModel): "Dataset seed filters keyed by field, applied before sampling. Accepted keys: harm_categories, data_types." ), ) - max_concurrency: int = Field(10, ge=1, le=100, description="Maximum concurrent operations") - max_retries: int = Field(0, ge=0, le=20, description="Maximum retry attempts on failure") + max_concurrency: int | None = Field(10, ge=1, le=100, description="Maximum concurrent operations") + max_retries: int | None = Field(0, ge=0, le=20, description="Maximum retry attempts on failure") include_baseline: bool | None = Field( None, description="Override the scenario baseline default; forbidden scenarios reject true" ) diff --git a/tests/unit/backend/test_scenario_resume.py b/tests/unit/backend/test_scenario_resume.py index e9b801a32b..95e492d214 100644 --- a/tests/unit/backend/test_scenario_resume.py +++ b/tests/unit/backend/test_scenario_resume.py @@ -405,6 +405,100 @@ async def fail_second_async(*, normalized_conversation: list[Message]) -> list[M return result +@pytest.mark.parametrize("field", ["max_concurrency", "max_retries"]) +@pytest.mark.parametrize("input_kind", ["omitted", "null", "explicit", "scenario_param", "top_level_wins"]) +async def test_numeric_launch_values_survive_failure_and_default_changes_async( + *, + resume_environment: tuple[ScenarioRunService, MockPromptTarget], + sqlite_instance: SQLiteMemory, + field: str, + input_kind: str, +) -> None: + service, target = resume_environment + scenario_default = 4 if field == "max_concurrency" else 0 + api_default = 10 if field == "max_concurrency" else 0 + explicit_value = 2 if field == "max_concurrency" else 1 + request_values: dict[str, Any] = {} + expected = api_default + if input_kind == "null": + request_values[field] = None + expected = scenario_default + elif input_kind == "explicit": + request_values[field] = explicit_value + expected = explicit_value + elif input_kind == "scenario_param": + request_values.update({field: None, "scenario_params": {field: explicit_value}}) + expected = explicit_value + elif input_kind == "top_level_wins": + request_values.update({field: explicit_value, "scenario_params": {field: explicit_value + 1}}) + expected = explicit_value + + prepared = await service._prepare_run_async( + request=RunScenarioRequest(scenario_name=_SCENARIO_NAME, target_name=_TARGET_NAME, **request_values) + ) + assert getattr(prepared.scenario, f"_{field}") == expected + original_send = target._send_prompt_to_target_async + + async def fail_second_async(*, normalized_conversation: list[Message]) -> list[Message]: + if normalized_conversation[-1].get_value() == _SECOND_OBJECTIVE: + raise RuntimeError("Synthetic target failure") + return await original_send(normalized_conversation=normalized_conversation) + + with patch.object(target, "_send_prompt_to_target_async", side_effect=fail_second_async): + with pytest.raises(ScenarioPartialFailureException): + await prepared.scenario.run_async() + run_id = prepared.scenario._scenario_result_id + assert run_id is not None + stored = await sqlite_instance.get_scenario_result_header_async(scenario_result_id=run_id) + assert stored is not None and stored.scenario_run_state == ScenarioRunState.FAILED + assert stored.metadata[_LAUNCH_REQUEST_METADATA_KEY][field] == expected + + changed_defaults = [ + parameter.model_copy(update={"default": expected + 3}) if parameter.name == field else parameter + for parameter in _OfflineResumeScenario._common_scenario_parameters() + ] + original_run = _OfflineResumeScenario.run_async + + async def run_with_saved_value_async(self: _OfflineResumeScenario) -> None: + assert getattr(self, f"_{field}") == expected + await original_run(self) + + with ( + patch.object(_OfflineResumeScenario, "_common_scenario_parameters", return_value=changed_defaults), + patch.object(_OfflineResumeScenario, "run_async", run_with_saved_value_async), + ): + response = await service.resume_run_async(scenario_result_id=run_id) + await _wait_for_idle_async(service) + assert response.scenario_result_id == run_id + resumed = await sqlite_instance.get_scenario_result_header_async(scenario_result_id=run_id) + assert resumed is not None and resumed.scenario_run_state == ScenarioRunState.COMPLETED + assert resumed.metadata[_LAUNCH_REQUEST_METADATA_KEY][field] == expected + + +@pytest.mark.parametrize("field", ["max_concurrency", "max_retries"]) +async def test_unsupported_numeric_launch_values_remain_omitted_on_resume_async( + *, + resume_environment: tuple[ScenarioRunService, MockPromptTarget], + sqlite_instance: SQLiteMemory, + field: str, +) -> None: + service, _ = resume_environment + declared = [parameter for parameter in _OfflineResumeScenario.supported_parameters() if parameter.name != field] + with patch.object(_OfflineResumeScenario, "supported_parameters", return_value=declared): + prepared = await service._prepare_run_async( + request=RunScenarioRequest(scenario_name=_SCENARIO_NAME, target_name=_TARGET_NAME) + ) + run_id = prepared.scenario._scenario_result_id + assert run_id is not None + stored = await sqlite_instance.get_scenario_result_header_async(scenario_result_id=run_id) + assert stored is not None + assert stored.metadata[_LAUNCH_REQUEST_METADATA_KEY][field] is None + restored = service._restore_launch_request(stored=stored) + assert getattr(restored, field) is None + resumed = await service._prepare_run_async(request=restored) + assert field not in resumed.scenario.params + + async def test_preparation_and_execution_share_sqlite_loop_resources_async( resume_environment: tuple[ScenarioRunService, MockPromptTarget], sqlite_instance: SQLiteMemory ) -> None: diff --git a/tests/unit/backend/test_scenario_run_service.py b/tests/unit/backend/test_scenario_run_service.py index 8d3e2c739a..da7e557ad2 100644 --- a/tests/unit/backend/test_scenario_run_service.py +++ b/tests/unit/backend/test_scenario_run_service.py @@ -178,6 +178,75 @@ def _make_request( ) +@pytest.mark.parametrize("supports_common_inputs", [True, False]) +async def test_prepare_and_restore_preserve_common_input_support_async( + patch_central_database: MagicMock, supports_common_inputs: bool +) -> None: + declared = Scam.supported_parameters() if supports_common_inputs else [] + registry = MagicMock(spec=ScenarioRegistry) + registry.get_class.return_value = Scam + registry.create_and_initialize_async = AsyncMock(return_value=MagicMock(spec=Scenario)) + service = ScenarioRunService() + request = RunScenarioRequest( + scenario_name="test.support", + **({"target_name": "my_target"} if supports_common_inputs else {}), + ) + with ( + patch.object(Scam, "supported_parameters", return_value=declared), + patch.object(ScenarioRegistry, "get_registry_singleton", return_value=registry), + patch.object(service, "_run_initializers_async", new_callable=AsyncMock), + patch.object(service._configuration_resolver, "resolve_target") as resolve_target, + ): + await service._prepare_run_async(request=request) + kwargs = registry.create_and_initialize_async.call_args.kwargs + saved = kwargs["initial_metadata"][_svc_mod._LAUNCH_REQUEST_METADATA_KEY] + stored = make_scenario_result( + scenario_name="test.support", + techniques=["prompt_sending"], + datasets=[], + attack_results={}, + metadata={_svc_mod._LAUNCH_REQUEST_METADATA_KEY: saved}, + ) + restored = service._restore_launch_request(stored=stored) + if supports_common_inputs: + resolve_target.assert_called_once() + assert kwargs["max_concurrency"] == restored.max_concurrency == 10 + assert kwargs["max_retries"] == restored.max_retries == 0 + assert restored.target_name == "my_target" + assert restored.include_baseline is True + else: + resolve_target.assert_not_called() + for name in ("objective_target", "max_concurrency", "max_retries", "include_baseline"): + assert name not in kwargs + assert restored.target_name is None + assert restored.max_concurrency is None + assert restored.max_retries is None + assert restored.include_baseline is None + assert restored.techniques is None + assert restored.dataset_names is None + assert restored.labels is None + registry.create_and_initialize_async.reset_mock() + await service._prepare_run_async(request=restored) + for name in ("objective_target", "max_concurrency", "max_retries", "include_baseline"): + assert name not in registry.create_and_initialize_async.call_args.kwargs + await service.close_async() + + +async def test_prepare_requires_target_for_scenarios_that_declare_it_async( + patch_central_database: MagicMock, +) -> None: + registry = MagicMock(spec=ScenarioRegistry) + registry.get_class.return_value = Scam + service = ScenarioRunService() + with ( + patch.object(ScenarioRegistry, "get_registry_singleton", return_value=registry), + patch.object(service, "_run_initializers_async", new_callable=AsyncMock), + pytest.raises(ValueError, match="requires an objective target"), + ): + await service._prepare_run_async(request=RunScenarioRequest(scenario_name="test.support")) + await service.close_async() + + @pytest.mark.parametrize( ("scenario_params", "rejected"), [ diff --git a/tests/unit/backend/test_scenario_service.py b/tests/unit/backend/test_scenario_service.py index 19dbdb2883..df78a77d48 100644 --- a/tests/unit/backend/test_scenario_service.py +++ b/tests/unit/backend/test_scenario_service.py @@ -47,6 +47,8 @@ override_default_adversarial_target, ) from pyrit.scenario.scenarios.airt.scam import Scam +from pyrit.scenario.scenarios.garak.exploitation import Exploitation +from pyrit.score import SubStringScorer from unit.mocks import MockPromptTarget if TYPE_CHECKING: @@ -147,6 +149,27 @@ def test_catalog_lists_only_external_scenario_parameters() -> None: summary = _metadata_to_registered_scenario(metadata=metadata) assert [parameter.name for parameter in summary.supported_parameters] == ["max_concurrency"] + assert summary.supported_parameter_names == ["dataset_config", "max_concurrency"] + + +@pytest.mark.usefixtures("patch_central_database") +@pytest.mark.parametrize("scenario_class", [Scam, Exploitation]) +def test_catalog_support_names_match_real_scenario_declarations(scenario_class: type[Scenario]) -> None: + with ( + patch.object(ScenarioRegistry, "_discover"), + patch.object(Scenario, "_get_default_objective_scorer", return_value=SubStringScorer(substring="test")), + override_default_adversarial_target(MockPromptTarget()), + ): + registry = ScenarioRegistry() + registry.register_class(scenario_class, name="test.support") + metadata = registry.get_registered_class_metadata("test.support") + assert metadata is not None + summary = _metadata_to_registered_scenario(metadata=metadata) + assert summary.supported_parameter_names == [parameter.name for parameter in scenario_class.supported_parameters()] + assert ("dataset_config" in summary.supported_parameter_names) is (scenario_class is Scam) + assert "dataset_config" not in [parameter.name for parameter in summary.supported_parameters] + assert "scenario_techniques" in summary.supported_parameter_names + assert "memory_labels" in summary.supported_parameter_names def test_catalog_describes_scenario_parameters_in_the_form_callers_send() -> None: @@ -1749,6 +1772,7 @@ def test_list_scenarios_with_items(self, client: TestClient) -> None: """Test that GET /api/scenarios/catalog returns scenario data.""" summary = RegisteredScenario( scenario_name="foundry.red_team_agent", + supported_parameter_names=[], scenario_type="RedTeamAgentScenario", description="Red team agent testing", description_markdown='', @@ -1853,6 +1877,7 @@ def test_get_scenario_returns_200(self, client: TestClient) -> None: """Test that GET /api/scenarios/catalog/{name} returns 200 when found.""" summary = RegisteredScenario( scenario_name="foundry.red_team_agent", + supported_parameter_names=[], scenario_type="RedTeamAgentScenario", description="Red team agent testing", default_technique="default", @@ -1994,6 +2019,7 @@ def test_get_scenario_with_dotted_name(self, client: TestClient) -> None: """Test that dotted scenario names (e.g., 'foundry.red_team_agent') work in path.""" summary = RegisteredScenario( scenario_name="garak.encoding", + supported_parameter_names=[], scenario_type="EncodingScenario", description="Encoding scenario", default_technique="all", diff --git a/tests/unit/cli/test_api_client.py b/tests/unit/cli/test_api_client.py index 7185fbf16d..38aa8deb32 100644 --- a/tests/unit/cli/test_api_client.py +++ b/tests/unit/cli/test_api_client.py @@ -343,6 +343,7 @@ def _scenario_payload(*, scenario_name: str = "s1") -> dict: "all_techniques": ["single_turn"], "default_datasets": [], "supported_parameters": [], + "supported_parameter_names": [], } diff --git a/tests/unit/cli/test_output.py b/tests/unit/cli/test_output.py index 35dcb7f00f..925820c173 100644 --- a/tests/unit/cli/test_output.py +++ b/tests/unit/cli/test_output.py @@ -43,6 +43,7 @@ def _make_scenario(**overrides) -> RegisteredScenario: "all_techniques": [], "default_datasets": [], "supported_parameters": [], + "supported_parameter_names": [], } defaults.update(overrides) return RegisteredScenario(**defaults) diff --git a/tests/unit/cli/test_pyrit_scan.py b/tests/unit/cli/test_pyrit_scan.py index e9fa072d37..43c4e1c565 100644 --- a/tests/unit/cli/test_pyrit_scan.py +++ b/tests/unit/cli/test_pyrit_scan.py @@ -423,6 +423,7 @@ def _mock_api_client(): all_techniques=[], default_datasets=[], supported_parameters=[], + supported_parameter_names=[], ) client.start_scenario_run_async.return_value = ScenarioRunSummary( scenario_result_id="test-id-123", @@ -1270,6 +1271,7 @@ def test_main_scenario_not_found_lists_available(self, mock_client_class, _mock_ mock_client.list_scenarios_async.return_value = [ RegisteredScenario( scenario_name="alt_a", + supported_parameter_names=[], scenario_type="X", description="", default_technique="", @@ -1279,6 +1281,7 @@ def test_main_scenario_not_found_lists_available(self, mock_client_class, _mock_ ), RegisteredScenario( scenario_name="alt_b", + supported_parameter_names=[], scenario_type="X", description="", default_technique="", @@ -1751,6 +1754,7 @@ def _build_mock_client(supported_params=None, status="COMPLETED"): client.list_scenarios_async.return_value = [ RegisteredScenario( scenario_name="foo", + supported_parameter_names=[], scenario_type="X", description="", default_technique="", @@ -1768,6 +1772,7 @@ def _build_mock_client(supported_params=None, status="COMPLETED"): all_techniques=[], default_datasets=[], supported_parameters=typed_params, + supported_parameter_names=[parameter.name for parameter in typed_params], ) client.start_scenario_run_async.return_value = ScenarioRunSummary( scenario_result_id="rid", diff --git a/tests/unit/cli/test_pyrit_shell.py b/tests/unit/cli/test_pyrit_shell.py index d678eaa8f3..5b204fe1c2 100644 --- a/tests/unit/cli/test_pyrit_shell.py +++ b/tests/unit/cli/test_pyrit_shell.py @@ -61,6 +61,7 @@ def mock_api_client(): all_techniques=[], default_datasets=[], supported_parameters=[], + supported_parameter_names=[], ) client.close_async = AsyncMock() client.__aenter__ = AsyncMock(return_value=client) @@ -75,6 +76,7 @@ def mock_api_client(): all_techniques=kw.get("all_techniques", []), default_datasets=kw.get("default_datasets", []), supported_parameters=kw.get("supported_parameters", []), + supported_parameter_names=[parameter.name for parameter in kw.get("supported_parameters", [])], ) # Suppress unused-import warning for datetime/timezone helpers used by tests. _ = (datetime, timezone)