From 3471807299cffb47f39706a0410f33aa9c0f8089 Mon Sep 17 00:00:00 2001 From: Richard Lundeen Date: Tue, 6 Oct 2026 23:08:17 -0700 Subject: [PATCH 01/15] Add runtime technique registry and GUI Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- doc/code/registry/0_registry.md | 114 ++++- doc/gui/0_gui.md | 43 +- frontend/e2e/registry.spec.ts | 96 +++- frontend/src/App.tsx | 2 + .../components/Parameters/ParameterField.tsx | 20 +- .../Parameters/ReferenceField.styles.ts | 9 + .../components/Parameters/ReferenceField.tsx | 83 +++ .../Registry/CreateTechniqueDialog.styles.ts | 10 + .../Registry/CreateTechniqueDialog.tsx | 247 +++++++++ .../Registry/RegistryLayout.test.tsx | 21 +- .../components/Registry/RegistryLayout.tsx | 4 +- .../Registry/TechniqueRegistry.styles.ts | 18 + .../Registry/TechniqueRegistry.test.tsx | 252 +++++++++ .../components/Registry/TechniqueRegistry.tsx | 192 +++++++ frontend/src/services/api.ts | 21 + frontend/src/types/index.ts | 57 +++ pyrit/backend/README.md | 38 ++ pyrit/backend/main.py | 2 + pyrit/backend/mappers/technique_mappers.py | 25 + pyrit/backend/models/techniques.py | 51 ++ pyrit/backend/routes/techniques.py | 69 +++ .../backend/services/scenario_run_service.py | 6 + pyrit/backend/services/scenario_service.py | 9 +- pyrit/backend/services/service_lifecycle.py | 2 + pyrit/backend/services/technique_service.py | 122 +++++ pyrit/models/technique_definition.py | 91 ++++ pyrit/registry/components/attack_registry.py | 7 + .../components/attack_technique_registry.py | 151 +++++- .../registry/components/scenario_registry.py | 20 + pyrit/registry/instance_registry.py | 11 + pyrit/registry/resolution.py | 238 ++++++++- pyrit/registry/technique_catalog.py | 54 ++ pyrit/registry/technique_inputs.py | 49 ++ .../scenario/core/attack_technique_factory.py | 53 ++ pyrit/scenario/core/scenario.py | 10 + .../scenario/scenarios/_dynamic_techniques.py | 4 +- .../scenarios/adaptive/text_adaptive.py | 6 +- pyrit/scenario/scenarios/airt/__init__.py | 4 + pyrit/scenario/scenarios/airt/cyber.py | 4 +- pyrit/scenario/scenarios/airt/leakage.py | 14 +- pyrit/scenario/scenarios/airt/multilingual.py | 14 +- .../scenario/scenarios/airt/rapid_response.py | 4 +- .../scenario/scenarios/benchmark/__init__.py | 4 + .../scenarios/benchmark/adversarial.py | 3 +- tests/unit/backend/test_scenario_service.py | 11 +- tests/unit/backend/test_technique_service.py | 243 +++++++++ .../registry/test_technique_definitions.py | 478 ++++++++++++++++++ 47 files changed, 2941 insertions(+), 45 deletions(-) create mode 100644 frontend/src/components/Parameters/ReferenceField.styles.ts create mode 100644 frontend/src/components/Parameters/ReferenceField.tsx create mode 100644 frontend/src/components/Registry/CreateTechniqueDialog.styles.ts create mode 100644 frontend/src/components/Registry/CreateTechniqueDialog.tsx create mode 100644 frontend/src/components/Registry/TechniqueRegistry.styles.ts create mode 100644 frontend/src/components/Registry/TechniqueRegistry.test.tsx create mode 100644 frontend/src/components/Registry/TechniqueRegistry.tsx create mode 100644 pyrit/backend/mappers/technique_mappers.py create mode 100644 pyrit/backend/models/techniques.py create mode 100644 pyrit/backend/routes/techniques.py create mode 100644 pyrit/backend/services/technique_service.py create mode 100644 pyrit/models/technique_definition.py create mode 100644 pyrit/registry/technique_catalog.py create mode 100644 pyrit/registry/technique_inputs.py create mode 100644 tests/unit/backend/test_technique_service.py create mode 100644 tests/unit/registry/test_technique_definitions.py diff --git a/doc/code/registry/0_registry.md b/doc/code/registry/0_registry.md index 0d0fc93c01..9975007b06 100644 --- a/doc/code/registry/0_registry.md +++ b/doc/code/registry/0_registry.md @@ -135,13 +135,123 @@ attack = AttackRegistry.get_registry_singleton().create_instance( Simple scalar inputs use the shared resolver. Pass live Python configuration objects, such as `AttackAdversarialConfig`, `AttackConverterConfig`, and `AttackScoringConfig`, for nested components. Advanced Python values, such as a -prompt normalizer or a parameter class, pass through unchanged. Nested JSON -attack recipes are not supported. +prompt normalizer or a parameter class, pass through unchanged. The shared +resolver also constructs explicitly declared structured configuration variants. +Use technique definitions for JSON-native, reusable attack configurations. An attack class implements the conversation algorithm. An attack technique factory selects and configures that class, converters, scorers, and seeds. `AttackTechniqueRegistry` continues to store those factories separately. +## Technique Definitions + +`TechniqueDefinition` is a JSON-native construction contract under +`pyrit.models.technique_definition`. It holds names, tags, attack arguments, +factory options, and typed technique seeds. Models do not import attacks, +resolve components, or select modules. + +`AttackTechniqueRegistry.build_from_definition()` resolves the registered attack +class and its declared inputs without registering or constructing an attack. +`register_definition()` also checks selector collisions and registers the factory +atomically. A failed definition leaves no partial entry. Existing Python factory +registration and live configuration values remain supported. +The selectors `all` and `default` are reserved. `types` is also a reserved name. +Names and tags cannot conflict with existing selectors or differ only by letter case. + +After initialization, register this definition through Python or send its JSON +representation to `POST /api/techniques`: + +```python +from pyrit.models.technique_definition import TechniqueDefinition +from pyrit.registry import AttackTechniqueRegistry + +definition = TechniqueDefinition.model_validate({ + "name": "encoded_example", + "description": "Encode the request with an existing converter.", + "tags": ["custom"], + "attack_type": "PromptSendingAttack", + "attack_args": { + "attack_converter_config": { + "type": "AttackConverterConfig", + "parameters": { + "request_converters": [{ + "type": "ConverterConfiguration", + "parameters": { + "converters": ["registered_base64", "registered_base64"], + "indexes_to_apply": [], + }, + }], + "response_converters": [], + }, + }, + "prepended_conversation_config": { + "type": "PrependedConversationConfig", + "parameters": {"apply_converters_to_roles": ["user"]}, + }, + }, + "factory_options": {"use_score_as_feedback": False}, + "seed_technique": { + "insertion_index": 0, + "prompt_placement": "prepend", + "seeds": [{ + "type": "SeedPrompt", + "parameters": { + "value": "A static prefix", + "role": "system", + "is_general_technique": True, + }, + }], + }, +}) +factory = AttackTechniqueRegistry.get_registry_singleton().register_definition(definition) +``` + +The converter name must already be registered. Converter order and duplicate +entries are retained. An explicit empty `indexes_to_apply` list applies the +converter to no pieces; it is not the same as omitting the field. + +Supported structured inputs include `AttackConverterConfig`, +`ConverterConfiguration`, `AttackScoringConfig`, `TAPAttackScoringConfig`, +and `PrependedConversationConfig`. Scoring configurations accept registered +scorer names, including ordered auxiliary-scorer lists. The factory keeps the +existing execution scorer-override policy (`warn`, `raise`, or `skip`). + +Seed groups accept declared `SeedPrompt` and `SeedSimulatedConversation` +variants. A simulated-conversation seed uses typed `SeedPrompt` values for +`adversarial_chat_system_prompt`, `simulated_target_system_prompt`, and optional +`next_message_system_prompt`. It also accepts `num_turns` and `sequence`. +Seed validators check general-technique flags, roles, sequence overlap, and +conversation constraints. Legacy server file-path inputs are not accepted by +this contract. Seed construction does not generate a conversation. + +For adversarial attacks, `factory_options` accepts an optional registered +`adversarial_chat` name and inline strings or typed `SeedPrompt` values for +system, seed, and per-turn prompts. The objective target belongs to execution +and cannot be supplied in `attack_args`. A default adversarial target is resolved +only when needed at execution. + +Custom registered attack classes use the same contract. A custom configuration +can inherit `StructuredParameterValue` and declare its allowed variants, or +trusted Python code can call `register_structured_input()` in +`pyrit.registry.resolution` to declare variants and nested component references. +Only server-declared types can be constructed. Clients cannot select modules, +execute code, or deserialize arbitrary objects. Callables and other Python-only +values need a Python initializer or programmatic factory. + +`GET /api/techniques/types` exposes the shared `Parameter` metadata and the +definition schema. The GUI is a basic subset of this contract; it is not the +backend's construction boundary. `GET /api/techniques` and detail responses +use `AttackTechniqueFactory.get_configuration()` for safe display. Identity +hashes and display projections are not lossless construction recipes. + +New definitions are **runtime only**. They are lost on restart or setup reset. +The instance registry exposes a mutation revision. Compatible scenario enums, +metadata, estimates, and summary caches use this revision and registry identity, +including direct `.instances.register()` calls and registry replacement. +Old scenario instances retain their technique snapshot. Active tasks are not +cancelled on catalog changes. Scenario filters and fixed local catalogs remain +in control of which techniques they support. + ## See Also - [Class Registries](1_class_registry.ipynb) - ScenarioRegistry, InitializerRegistry diff --git a/doc/gui/0_gui.md b/doc/gui/0_gui.md index fafd4fb397..f9e71685c4 100644 --- a/doc/gui/0_gui.md +++ b/doc/gui/0_gui.md @@ -1,6 +1,6 @@ # PyRIT GUI (CoPyRIT) -CoPyRIT is a web-based graphical interface for PyRIT built with React and Fluent UI. It provides an interactive way to run attacks, configure targets and converters, and view results — all from a browser. +CoPyRIT is a web-based graphical interface for PyRIT built with React and Fluent UI. It provides an interactive way to run attacks, configure targets, converters, and techniques, and view results — all from a browser. ## Getting Started @@ -414,6 +414,47 @@ In active runs and saved scenario results, **Atomic attack groups** defaults to Until you expand or collapse the section, its default follows the current group count as progress loads. Once you choose, the section keeps your choice during progress updates for the same run, even if the count crosses 20. Opening a different run resets to that run's count-based default. +### Technique Registry + +Open **Registry > Techniques**, or go to `/registry/techniques`. Targets and +Converters keep their existing registry URLs. + +The table shows every registered technique factory, including advanced factories +from Python initializers. Search by name, description, attack type, or tag. Use +the attack-type and tag filters to reduce the list. **Details** shows safe +settings. Target credentials are not shown. These settings are a display +projection, not a recipe that can reconstruct a factory from its identity hash. + +Select **New technique** to configure an existing attack class. Set a unique +name, a description, and optional comma-separated tags. Names and tags start +with a letter and use letters, digits, and underscores. `all` and `default` +are reserved selectors. `types` is also a reserved technique name. Names cannot differ only by letter case or conflict with an +aggregate tag or a scenario-local technique name. No `core`, `light`, or +`default` tag is added. + +The basic form supports scalar settings and supported structured variants. +Request and response converters run in the displayed order. You can add the +same registered converter more than once, move an entry, or remove an entry. +For an adversarial attack, select a registered adversarial target and set inline +system, seed, and per-turn prompts. If the adversarial target is not set, PyRIT +resolves the default at execution. Select the objective target when you run a +scenario, not when you create a technique. + +The form does not edit complex seeds or simulated conversations. It marks +unsupported inputs and blocks creation when a required input cannot be set. +Use the [typed REST contract](../code/registry/0_registry.md#technique-definitions) +or a Python initializer for advanced settings and Python-only values. + +Creation does not construct an attack or send prompts. A new technique becomes +selectable in a compatible scenario without a restart, including after scenario +metadata or estimates have been cached. Scenario filters and fixed technique +catalogs still apply. Existing scenario snapshots and active runs do not change. + +**Runtime only:** New techniques are lost when PyRIT restarts or is +reinitialized. There is no persistence, edit, or delete operation in this pane. +The pane lists only the active registered pool. It does not load more techniques +or change the configured initializers. + ### Target Configuration The Configuration view manages the targets available for attacks. diff --git a/frontend/e2e/registry.spec.ts b/frontend/e2e/registry.spec.ts index 0d54ff5154..6189b5b497 100644 --- a/frontend/e2e/registry.spec.ts +++ b/frontend/e2e/registry.spec.ts @@ -1,5 +1,99 @@ import { expect, test, type Page } from "./_fixtures"; -import { mockVersion } from "./_compatibility"; +import { compatibilityHeaders, mockVersion } from "./_compatibility"; + +test("creates and selects a runtime technique with the real backend @seeded", async ({ page, request }) => { + test.setTimeout(120_000); + const headers = compatibilityHeaders(); + for (const name of ["objective_scorer_chat", "adversarial_chat", "registry_local_objective"]) { + const existing = await request.get(`/api/targets/${name}`, { headers }); + if (existing.status() === 404) { + const created = await request.post("/api/targets", { headers, data: name === "registry_local_objective" + ? { name, type: "TextTarget", params: {} } + : { name, type: "OpenAIChatTarget", auth_mode: "api_key", params: { + endpoint: "http://127.0.0.1:9/v1", model_name: "local-test", api_key: "local-test-only", + } } }); + expect(created.ok(), await created.text()).toBe(true); + } else { + expect(existing.ok(), await existing.text()).toBe(true); + } + } + const warm = await request.get("/api/scenarios/catalog/airt.rapid_response", { headers }); + expect(warm.ok(), await warm.text()).toBe(true); + const before = await warm.json(); + const name = `browser_${Date.now()}`; + expect(before.all_techniques).not.toContain(name); + let replacementGeneration: string | undefined; + await page.route("**/api/runtime", async (route) => { + const response = await route.fetch(); + const readiness = await response.json(); + await route.fulfill({ json: { ...readiness, generation: replacementGeneration ?? readiness.generation } }); + }); + + const promptRequests: string[] = []; + page.on("request", (outgoing) => { + if (outgoing.method() === "POST" && /\/api\/(?:message-sends|attacks\/[^/]+\/messages|scenarios\/runs)(?:\/|\?|$)/.test(outgoing.url())) { + promptRequests.push(outgoing.url()); + } + }); + await page.goto("/registry/techniques"); + await expect(page.getByRole("tab", { name: "Techniques" })).toHaveAttribute("aria-selected", "true"); + await expect(page.getByRole("table", { name: "Registered techniques" })).toBeVisible(); + await page.getByRole("button", { name: "New technique" }).click(); + await page.getByRole("textbox", { name: "Registry name" }).fill(name); + await page.getByRole("textbox", { name: "Description" }).fill("Local browser technique"); + await page.getByRole("textbox", { name: "Tags" }).fill("browser_test"); + await page.getByRole("combobox", { name: "Attack type", exact: true }).selectOption("PromptSendingAttack"); + const create = page.waitForResponse((response) => response.url().endsWith("/api/techniques") && response.request().method() === "POST"); + await page.getByRole("button", { name: "Add technique" }).click(); + expect((await create).status()).toBe(201); + await expect(page.getByRole("dialog")).not.toBeVisible(); + await page.getByRole("textbox", { name: "Search techniques" }).fill(name); + await expect(page.getByRole("cell", { name, exact: true })).toBeVisible(); + await page.getByRole("button", { name: `Details for ${name}` }).click(); + await expect(page.getByText("These are safe display settings, not a reconstruction recipe.")).toBeVisible(); + await page.getByRole("button", { name: "Close", exact: true }).click(); + + let finishCreate: (() => void) | undefined; + const pendingCreate = new Promise((resolve) => { finishCreate = resolve; }); + let markCreateStarted: (() => void) | undefined; + const createStarted = new Promise((resolve) => { markCreateStarted = resolve; }); + const original = await request.get(`/api/techniques/${name}`, { headers }); + const originalTechnique = await original.json(); + await page.route("**/api/techniques", async (route) => { + if (route.request().method() !== "POST") { + await route.continue(); + return; + } + markCreateStarted?.(); + await pendingCreate; + await route.fulfill({ status: 201, json: { ...originalTechnique, name: "old_runtime_response" } }); + }); + await page.getByRole("button", { name: "New technique" }).click(); + await page.getByRole("textbox", { name: "Registry name" }).fill("old_runtime_response"); + await page.getByRole("combobox", { name: "Attack type", exact: true }).selectOption("PromptSendingAttack"); + await page.getByRole("button", { name: "Add technique" }).click(); + await createStarted; + replacementGeneration = "registry-replacement-test"; + await expect(page.getByRole("dialog")).not.toBeVisible(); + await expect(page.getByRole("button", { name: "New technique" })).toBeFocused(); + await expect(page.getByRole("table", { name: "Registered techniques" })).toBeVisible(); + const staleResponse = page.waitForResponse((response) => response.url().endsWith("/api/techniques") && response.request().method() === "POST"); + finishCreate?.(); + await staleResponse; + await expect(page.getByRole("cell", { name: "old_runtime_response", exact: true })).toHaveCount(0); + + const current = await request.get("/api/scenarios/catalog/airt.rapid_response", { headers }); + expect(current.ok(), await current.text()).toBe(true); + const after = await current.json(); + expect(after.all_techniques).toContain(name); + expect(after.default_techniques).toEqual(before.default_techniques); + await page.goto("/scanner/airt.rapid_response"); + const checkbox = page.getByRole("checkbox", { name, exact: true }); + await expect(checkbox).toBeVisible(); + await checkbox.check(); + await expect(checkbox).toBeChecked(); + expect(promptRequests).toEqual([]); +}); interface RegisteredConverter { converter_id: string; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e110b0e8c1..fed3129f4e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,6 +12,7 @@ import AttackNotFound from './components/Chat/AttackNotFound' import Home from './components/Home/Home' import TargetConfig from './components/Config/TargetConfig' import ConverterRegistry from './components/Registry/ConverterRegistry' +import TechniqueRegistry from './components/Registry/TechniqueRegistry' import RegistryLayout from './components/Registry/RegistryLayout' import Configuration from './components/Configuration/Configuration' import AttackHistory from './components/History/AttackHistory' @@ -730,6 +731,7 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { } /> } /> + } /> } /> } /> diff --git a/frontend/src/components/Parameters/ParameterField.tsx b/frontend/src/components/Parameters/ParameterField.tsx index b6c646d95f..e855bc7525 100644 --- a/frontend/src/components/Parameters/ParameterField.tsx +++ b/frontend/src/components/Parameters/ParameterField.tsx @@ -5,8 +5,9 @@ import { Select, } from '@fluentui/react-components' -import type { Parameter } from '@/types' +import type { Parameter, ParameterReferenceOptions } from '@/types' +import ReferenceField from './ReferenceField' import { useParameterFieldStyles } from './ParameterField.styles' import { getInitialFormValues, @@ -26,6 +27,7 @@ export interface ParameterFieldProps { showRequiredError?: boolean /** Prefix for `data-testid` attributes. Defaults to `'param'` (e.g. `param-`). */ testIdPrefix?: string + referenceOptions?: ParameterReferenceOptions } /** @@ -46,12 +48,27 @@ export default function ParameterField({ allowEmptyList = false, showRequiredError = false, testIdPrefix = 'param', + referenceOptions, }: ParameterFieldProps) { const styles = useParameterFieldStyles() const kind = getParameterControlKind(parameter) const label = parameter.required ? `${parameter.name} *` : parameter.name const testId = `${testIdPrefix}-${parameter.name}` + if (parameter.reference_type && referenceOptions) { + return ( + onChange(parameter.name, next)} + /> + ) + } + if (kind === 'structured') { const current = isStructuredParameterFormValue(value) ? value : { type: '', values: {} } return ( @@ -84,6 +101,7 @@ export default function ParameterField({ value={current.values[nested.name] ?? ''} disabled={disabled} allowEmptyList + referenceOptions={referenceOptions} testIdPrefix={`${testIdPrefix}-${parameter.name}`} onChange={(name, nestedValue) => onChange(parameter.name, { ...current, diff --git a/frontend/src/components/Parameters/ReferenceField.styles.ts b/frontend/src/components/Parameters/ReferenceField.styles.ts new file mode 100644 index 0000000000..7407928fec --- /dev/null +++ b/frontend/src/components/Parameters/ReferenceField.styles.ts @@ -0,0 +1,9 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight } from '@/styles/touchTargets' + +export const useReferenceFieldStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalS }, + row: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', gap: tokens.spacingHorizontalS }, + action: { ...mobileTouchTargetHeight }, +}) diff --git a/frontend/src/components/Parameters/ReferenceField.tsx b/frontend/src/components/Parameters/ReferenceField.tsx new file mode 100644 index 0000000000..dbc50a2fcc --- /dev/null +++ b/frontend/src/components/Parameters/ReferenceField.tsx @@ -0,0 +1,83 @@ +import { useId, useState } from 'react' + +import { Button, Field, Select, Text } from '@fluentui/react-components' + +import type { RegistryReferenceOption } from '@/types' + +import { useReferenceFieldStyles } from './ReferenceField.styles' + +interface ReferenceFieldProps { + label: string + options: RegistryReferenceOption[] + value: string | string[] + multiple?: boolean + disabled?: boolean + hint?: string + onChange: (value: string | string[]) => void +} + +/** Ordered registry references. Repeated entries are intentional. */ +export default function ReferenceField({ + label, options, value, multiple = false, disabled = false, hint, onChange, +}: ReferenceFieldProps) { + const styles = useReferenceFieldStyles() + const [pending, setPending] = useState('') + const [entryIds, setEntryIds] = useState([]) + const id = useId() + const [nextId, setNextId] = useState(0) + const selected = Array.isArray(value) ? value : [] + + const move = (index: number, delta: number): void => { + const next = [...selected] + const ids = selected.map((_, position) => entryIds[position] ?? `${id}-initial-${position}`) + ;[next[index], next[index + delta]] = [next[index + delta], next[index]] + ;[ids[index], ids[index + delta]] = [ids[index + delta], ids[index]] + setEntryIds(ids) + onChange(next) + } + + return ( +
+ + + + {multiple && ( + <> + + {selected.map((name, index) => ( +
+ {index + 1}. {name} + + + +
+ ))} + + )} +
+ ) +} diff --git a/frontend/src/components/Registry/CreateTechniqueDialog.styles.ts b/frontend/src/components/Registry/CreateTechniqueDialog.styles.ts new file mode 100644 index 0000000000..5049ba9f3b --- /dev/null +++ b/frontend/src/components/Registry/CreateTechniqueDialog.styles.ts @@ -0,0 +1,10 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight } from '@/styles/touchTargets' + +export const useCreateTechniqueDialogStyles = makeStyles({ + surface: { width: 'min(640px, 90vw)', maxWidth: '90vw', maxHeight: '90vh' }, + content: { overflowY: 'auto', maxHeight: '65vh' }, + form: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalM }, + action: { ...mobileTouchTargetHeight }, +}) diff --git a/frontend/src/components/Registry/CreateTechniqueDialog.tsx b/frontend/src/components/Registry/CreateTechniqueDialog.tsx new file mode 100644 index 0000000000..0fc347e05c --- /dev/null +++ b/frontend/src/components/Registry/CreateTechniqueDialog.tsx @@ -0,0 +1,247 @@ +import { useEffect, useRef, useState } from 'react' + +import { + Button, Dialog, DialogActions, DialogBody, DialogContent, DialogSurface, DialogTitle, + Field, Input, MessageBar, MessageBarBody, Select, Spinner, Text, Textarea, +} from '@fluentui/react-components' + +import ParameterField from '@/components/Parameters/ParameterField' +import ReferenceField from '@/components/Parameters/ReferenceField' +import { buildParametersFromForm, getInitialFormValues, type ParameterFormValue } from '@/components/Parameters/parameterForm' +import { convertersApi, scorersApi, targetsApi, techniquesApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { CreateTechniqueRequest, PaginationInfo, Parameter, ParameterReferenceOptions, TechniqueTypeEntry } from '@/types' + +import { useCreateTechniqueDialogStyles } from './CreateTechniqueDialog.styles' + +const SCALAR_TYPES = new Set(['str', 'int', 'float', 'bool', 'list[str]', 'list[int]', 'list[float]', 'list[bool]']) +const NAME_PATTERN = /^[A-Za-z][A-Za-z0-9_]{0,63}$/ +const RESERVED_SELECTORS = new Set(['all', 'default']) + +function canConfigure(parameter: Parameter): boolean { + if (parameter.reference_type) return parameter.reference_type !== 'scenario' + if (parameter.variants) { + return !parameter.is_list && Object.values(parameter.variants).some((parameters) => + parameters.every((nested) => !nested.required || canConfigure(nested))) + } + return SCALAR_TYPES.has(parameter.type_name) || Boolean(parameter.choices?.length) +} + +async function loadReferencePages( + load: (cursor?: string) => Promise<{ items: T[]; pagination: PaginationInfo }>, +): Promise { + const items: T[] = [] + const seen = new Set() + let cursor: string | undefined + do { + const page = await load(cursor) + items.push(...page.items) + if (!page.pagination.has_more) return items + const next = page.pagination.next_cursor + if (!next || seen.has(next)) throw new Error('The registry returned an invalid page cursor.') + seen.add(next) + cursor = next + } while (cursor) + return items +} + +function guiParameters(parameters: Parameter[]): Parameter[] { + return parameters.filter(canConfigure).map((parameter) => parameter.variants ? { + ...parameter, + variants: Object.fromEntries(Object.entries(parameter.variants) + .filter(([, nested]) => nested.every((entry) => !entry.required || canConfigure(entry))) + .map(([name, nested]) => [name, guiParameters(nested)])), + } : parameter) +} + +interface CreateTechniqueDialogProps { + onClose: () => void + onCreated: () => void +} + +/** A basic form; the REST definition contract also supports advanced seeds and configurations. */ +export default function CreateTechniqueDialog({ onClose, onCreated }: CreateTechniqueDialogProps) { + const styles = useCreateTechniqueDialogStyles() + const [types, setTypes] = useState([]) + const [references, setReferences] = useState({ target: [], converter: [], scorer: [], scenario: [] }) + const [selectedType, setSelectedType] = useState('') + const [name, setName] = useState('') + const [description, setDescription] = useState('') + const [tags, setTags] = useState('') + const [values, setValues] = useState>({}) + const [requestConverters, setRequestConverters] = useState([]) + const [responseConverters, setResponseConverters] = useState([]) + const [adversarialTarget, setAdversarialTarget] = useState('') + const [systemPrompt, setSystemPrompt] = useState('') + const [seedPrompt, setSeedPrompt] = useState('') + const [turnPrompt, setTurnPrompt] = useState('') + const [loading, setLoading] = useState(true) + const [submitting, setSubmitting] = useState(false) + const [error, setError] = useState(null) + const [retry, setRetry] = useState(0) + const active = useRef(true) + const errorRef = useRef(null) + const [submitError, setSubmitError] = useState(false) + + useEffect(() => { + active.current = true + return () => { active.current = false } + }, []) + + useEffect(() => { + let ignore = false + void Promise.resolve().then(async () => { + if (ignore) return + setLoading(true) + setError(null) + try { + const [metadata, targets, converters, scorers] = await Promise.all([ + techniquesApi.listTypes(), + loadReferencePages((cursor) => targetsApi.listTargets(200, cursor)), + convertersApi.listConverters(), + loadReferencePages((cursor) => scorersApi.listScorers(cursor)), + ]) + if (ignore) return + setTypes(metadata.items) + setReferences({ + target: targets.map((entry) => ({ name: entry.target_registry_name, type: entry.identifier.class_name })), + converter: converters.items.map((entry) => ({ name: entry.converter_id, type: entry.identifier.class_name })), + scorer: scorers.map((entry) => ({ name: entry.scorer_registry_name, type: entry.identifier.class_name })), + scenario: [], + }) + } catch (err) { + if (!ignore) setError(toApiError(err).detail) + } finally { + if (!ignore) setLoading(false) + } + }) + return () => { ignore = true } + }, [retry]) + + useEffect(() => { + if (submitError && error) errorRef.current?.focus() + }, [error, submitError]) + + const selected = types.find((entry) => entry.attack_type === selectedType) + const attackParameters = selected?.parameters.filter((parameter) => parameter.name !== 'attack_converter_config') ?? [] + const unsupportedRequired = attackParameters.filter((parameter) => parameter.required && !canConfigure(parameter)) + const parameters = guiParameters(attackParameters) + + const submit = async (): Promise => { + const fail = (message: string): void => { setError(message); setSubmitError(true) } + const parsedTags = tags.split(',').map((tag) => tag.trim()).filter(Boolean) + if (!NAME_PATTERN.test(name) || RESERVED_SELECTORS.has(name.toLowerCase()) || name.toLowerCase() === 'types') { + fail('Use a unique name that starts with a letter. Use letters, digits, and underscores. Do not use all, default, or types.') + return + } + if (parsedTags.some((tag) => !NAME_PATTERN.test(tag) || RESERVED_SELECTORS.has(tag.toLowerCase())) + || new Set(parsedTags.map((tag) => tag.toLowerCase())).size !== parsedTags.length) { + fail('Use unique tags with letters, digits, and underscores. Do not use all or default.') + return + } + if (!selected || unsupportedRequired.length) { + fail('This attack needs inputs that cannot be set in this form. Use the REST API or a Python initializer.') + return + } + const result = buildParametersFromForm(parameters, values) + if (!result.ok) { fail(result.error); return } + const request: CreateTechniqueRequest = { + name, description, tags: parsedTags, attack_type: selectedType, + attack_args: result.parameters ?? {}, factory_options: {}, + } + if (selected.supports_converters && (requestConverters.length || responseConverters.length)) { + request.attack_args.attack_converter_config = { + type: 'AttackConverterConfig', parameters: { + request_converters: requestConverters.map((converter) => ({ type: 'ConverterConfiguration', parameters: { converters: [converter] } })), + response_converters: responseConverters.map((converter) => ({ type: 'ConverterConfiguration', parameters: { converters: [converter] } })), + }, + } + } + if (selected.supports_adversarial) { + if (adversarialTarget) request.factory_options.adversarial_chat = adversarialTarget + if (systemPrompt) request.factory_options.adversarial_system_prompt = systemPrompt + if (seedPrompt) request.factory_options.adversarial_seed_prompt = seedPrompt + if (turnPrompt) request.factory_options.adversarial_prompt_template = turnPrompt + } + setSubmitting(true) + setError(null) + setSubmitError(false) + try { + await techniquesApi.createTechnique(request) + if (active.current) onCreated() + } catch (err) { + if (active.current) fail(toApiError(err).detail) + } finally { + if (active.current) setSubmitting(false) + } + } + + return ( + { if (!data.open) onClose() }}> + + + New technique + +
+ Runtime only. Restart or reinitialize PyRIT to remove runtime additions. + {loading && } + {error && {error}} + {!loading && types.length === 0 && } + {!loading && types.length > 0 && ( + <> + setName(data.value)} /> +