diff --git a/doc/code/registry/0_registry.md b/doc/code/registry/0_registry.md index 0d0fc93c01..4f7889d685 100644 --- a/doc/code/registry/0_registry.md +++ b/doc/code/registry/0_registry.md @@ -135,8 +135,8 @@ attack = AttackRegistry.get_registry_singleton().create_instance( Simple scalar inputs use the shared resolver. Pass live Python configuration objects, such as `AttackAdversarialConfig`, `AttackConverterConfig`, and `AttackScoringConfig`, for nested components. Advanced Python values, such as a -prompt normalizer or a parameter class, pass through unchanged. Nested JSON -attack recipes are not supported. +prompt normalizer or a parameter class, pass through unchanged. The shared +resolver also constructs explicitly declared structured configuration variants. An attack class implements the conversation algorithm. An attack technique factory selects and configures that class, converters, scorers, and seeds. diff --git a/doc/gui/0_gui.md b/doc/gui/0_gui.md index d23fe60a8f..33e18dd0a7 100644 --- a/doc/gui/0_gui.md +++ b/doc/gui/0_gui.md @@ -1,6 +1,6 @@ # PyRIT GUI (CoPyRIT) -CoPyRIT is a web-based graphical interface for PyRIT built with React and Fluent UI. It provides an interactive way to run attacks, configure targets and converters, and view results — all from a browser. +CoPyRIT is a web-based graphical interface for PyRIT built with React and Fluent UI. It provides an interactive way to run attacks, configure targets, converters, and techniques, and view results — all from a browser. ## Getting Started @@ -469,6 +469,21 @@ In active runs and saved scenario results, **Atomic attack groups** defaults to Until you expand or collapse the section, its default follows the current group count as progress loads. Once you choose, the section keeps your choice during progress updates for the same run, even if the count crosses 20. Opening a different run resets to that run's count-based default. +### Technique Registry + +Open **Registry > Techniques**, or go to `/registry/techniques`. Targets and +Converters keep their existing registry URLs. + +Search or filter registered techniques, then select **Details** to inspect one. +Select **New technique** to configure an existing attack with basic settings, +ordered converters, and optional adversarial prompts. The objective target is +selected when you run a scenario. Seeds and conversation settings need a Python +initializer. Creation does not run an attack. + +New techniques are available in compatible scenarios and are lost on restart or +reinitialization. The pane does not edit or delete techniques. +See the [backend README](../../pyrit/backend/README.md#techniques) for the API. + ### Target Configuration The Configuration view manages the targets available for attacks. diff --git a/frontend/e2e/registry.spec.ts b/frontend/e2e/registry.spec.ts index 0d54ff5154..7ee3085052 100644 --- a/frontend/e2e/registry.spec.ts +++ b/frontend/e2e/registry.spec.ts @@ -1,5 +1,149 @@ import { expect, test, type Page } from "./_fixtures"; -import { mockVersion } from "./_compatibility"; +import { compatibilityHeaders, mockVersion } from "./_compatibility"; + +test("creates and selects a runtime technique with the real backend @seeded", async ({ page, request }) => { + test.setTimeout(120_000); + const headers = compatibilityHeaders(); + for (const name of ["objective_scorer_chat", "adversarial_chat", "registry_local_objective"]) { + const existing = await request.get(`/api/targets/${name}`, { headers }); + if (existing.status() === 404) { + const created = await request.post("/api/targets", { headers, data: name === "registry_local_objective" + ? { name, type: "TextTarget", params: {} } + : { name, type: "OpenAIChatTarget", auth_mode: "api_key", params: { + endpoint: "http://127.0.0.1:9/v1", model_name: "local-test", api_key: "local-test-only", + } } }); + expect(created.ok(), await created.text()).toBe(true); + } else { + expect(existing.ok(), await existing.text()).toBe(true); + } + } + const warm = await request.get("/api/scenarios/catalog/airt.rapid_response", { headers }); + expect(warm.ok(), await warm.text()).toBe(true); + const before = await warm.json(); + const name = `browser_${Date.now()}`; + expect(before.all_techniques).not.toContain(name); + const converterName = `${name}_b64`; + const converter = await request.post("/api/converters", { + headers, data: { name: converterName, type: "Base64Converter", params: {} }, + }); + expect(converter.ok(), await converter.text()).toBe(true); + let replacementGeneration: string | undefined; + await page.route("**/api/runtime", async (route) => { + const response = await route.fetch(); + const readiness = await response.json(); + await route.fulfill({ json: { ...readiness, generation: replacementGeneration ?? readiness.generation } }); + }); + + const promptRequests: string[] = []; + const detailRequests: string[] = []; + page.on("request", (outgoing) => { + if (outgoing.method() === "POST" && /\/api\/(?:message-sends|attacks\/[^/]+\/messages|scenarios\/runs)(?:\/|\?|$)/.test(outgoing.url())) { + promptRequests.push(outgoing.url()); + } + if (outgoing.method() === "GET" && new URL(outgoing.url()).pathname === `/api/techniques/${name}`) { + detailRequests.push(outgoing.url()); + } + }); + await page.goto("/registry/techniques"); + await expect(page.getByRole("tab", { name: "Techniques" })).toHaveAttribute("aria-selected", "true"); + await expect(page.getByText("Named configurations of existing attack techniques", { exact: true })).toBeVisible(); + await expect(page.getByText(/Runtime only|No attack runs when you add a technique/)).toHaveCount(0); + await expect(page.getByRole("table", { name: "Registered techniques" })).toBeVisible(); + const builtinCalls = { + crescendo_simulated: "AttackTechniqueFactory.with_simulated_conversation(\n" + + " name='crescendo_simulated',\n" + + " description='Escalates gradually over a simulated conversation toward the objective.',\n" + + " technique_tags=['single_turn'],\n)", + tap: "AttackTechniqueFactory(\n" + + " name='tap',\n" + + " attack_class=TreeOfAttacksWithPruningAttack,\n" + + " description='Explores a tree of adversarial prompts, pruning weak branches to refine the attack.',\n" + + " technique_tags=['multi_turn'],\n)", + }; + for (const [builtinName, statement] of Object.entries(builtinCalls)) { + await page.getByRole("button", { name: `Details for ${builtinName}` }).click(); + await expect(page.getByLabel("Technique creation call", { exact: true })).toHaveText(statement); + await expect(page.getByRole("heading", { name: "Configuration" })).toHaveCount(0); + await page.getByRole("button", { name: "Close", exact: true }).click(); + } + await page.getByRole("button", { name: "New technique" }).click(); + await page.getByRole("textbox", { name: "Registry name" }).fill(name); + await page.getByRole("textbox", { name: "Description" }).fill("Local browser technique"); + await page.getByRole("textbox", { name: "Tags" }).fill("browser_test"); + await page.getByRole("combobox", { name: "Attack type", exact: true }).selectOption("PromptSendingAttack"); + await expect(page.getByRole("dialog")).not.toContainText(/Implementation of|This form shows|Runtime only|prompt_normalizer/); + await page.getByLabel("max_attempts_on_failure").fill("0"); + await page.getByRole("combobox", { name: "Request converters", exact: true }).selectOption(converterName); + await page.getByRole("button", { name: "Add to Request converters", exact: true }).click(); + await page.getByRole("button", { name: "Add to Request converters", exact: true }).click(); + const create = page.waitForResponse((response) => response.url().endsWith("/api/techniques") && response.request().method() === "POST"); + await page.getByRole("button", { name: "Add technique" }).click(); + const created = await create; + expect(created.status()).toBe(201); + const createdTechnique = await created.json(); + const body = created.request().postDataJSON(); + expect(body).toMatchObject({ + name, type: "PromptSendingAttack", params: { max_attempts_on_failure: 0 }, + request_converters: [converterName, converterName], response_converters: [], + }); + expect(body).not.toHaveProperty("factory_options"); + expect(body).not.toHaveProperty("seed_technique"); + await expect(page.getByRole("dialog")).not.toBeVisible(); + await page.getByRole("textbox", { name: "Search techniques" }).fill(name); + await expect(page.getByRole("cell", { name, exact: true })).toBeVisible(); + await page.getByRole("button", { name: `Details for ${name}` }).click(); + const creationCall = page.getByLabel("Technique creation call", { exact: true }); + await expect(creationCall).toHaveText(createdTechnique.creation_statement); + await expect(creationCall).toContainText("AttackTechniqueFactory("); + await expect(creationCall).toContainText("'max_attempts_on_failure': 0"); + expect(createdTechnique).not.toHaveProperty("evaluation_identifier"); + expect(createdTechnique).not.toHaveProperty("configuration"); + await expect(page.getByRole("heading", { name: "Configuration" })).toHaveCount(0); + await page.getByRole("button", { name: "Close", exact: true }).click(); + await expect(page.getByRole("button", { name: `Details for ${name}` })).toBeFocused(); + expect(detailRequests).toEqual([]); + + let finishCreate: (() => void) | undefined; + const pendingCreate = new Promise((resolve) => { finishCreate = resolve; }); + let markCreateStarted: (() => void) | undefined; + const createStarted = new Promise((resolve) => { markCreateStarted = resolve; }); + const original = await request.get(`/api/techniques/${name}`, { headers }); + const originalTechnique = await original.json(); + await page.route("**/api/techniques", async (route) => { + if (route.request().method() !== "POST") { + await route.continue(); + return; + } + markCreateStarted?.(); + await pendingCreate; + await route.fulfill({ status: 201, json: { ...originalTechnique, name: "old_runtime_response" } }); + }); + await page.getByRole("button", { name: "New technique" }).click(); + await page.getByRole("textbox", { name: "Registry name" }).fill("old_runtime_response"); + await page.getByRole("combobox", { name: "Attack type", exact: true }).selectOption("PromptSendingAttack"); + await page.getByRole("button", { name: "Add technique" }).click(); + await createStarted; + replacementGeneration = "registry-replacement-test"; + await expect(page.getByRole("dialog")).not.toBeVisible(); + await expect(page.getByRole("button", { name: "New technique" })).toBeFocused(); + await expect(page.getByRole("table", { name: "Registered techniques" })).toBeVisible(); + const staleResponse = page.waitForResponse((response) => response.url().endsWith("/api/techniques") && response.request().method() === "POST"); + finishCreate?.(); + await staleResponse; + await expect(page.getByRole("cell", { name: "old_runtime_response", exact: true })).toHaveCount(0); + + const current = await request.get("/api/scenarios/catalog/airt.rapid_response", { headers }); + expect(current.ok(), await current.text()).toBe(true); + const after = await current.json(); + expect(after.all_techniques).toContain(name); + expect(after.default_techniques).toEqual(before.default_techniques); + await page.goto("/scanner/airt.rapid_response"); + const checkbox = page.getByRole("checkbox", { name, exact: true }); + await expect(checkbox).toBeVisible(); + await checkbox.check(); + await expect(checkbox).toBeChecked(); + expect(promptRequests).toEqual([]); +}); interface RegisteredConverter { converter_id: string; diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e110b0e8c1..fed3129f4e 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -12,6 +12,7 @@ import AttackNotFound from './components/Chat/AttackNotFound' import Home from './components/Home/Home' import TargetConfig from './components/Config/TargetConfig' import ConverterRegistry from './components/Registry/ConverterRegistry' +import TechniqueRegistry from './components/Registry/TechniqueRegistry' import RegistryLayout from './components/Registry/RegistryLayout' import Configuration from './components/Configuration/Configuration' import AttackHistory from './components/History/AttackHistory' @@ -730,6 +731,7 @@ function AppContent({ operatorAlias }: { operatorAlias: string | null }) { } /> } /> + } /> } /> } /> diff --git a/frontend/src/components/Parameters/ReferenceField.styles.ts b/frontend/src/components/Parameters/ReferenceField.styles.ts new file mode 100644 index 0000000000..7407928fec --- /dev/null +++ b/frontend/src/components/Parameters/ReferenceField.styles.ts @@ -0,0 +1,9 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight } from '@/styles/touchTargets' + +export const useReferenceFieldStyles = makeStyles({ + root: { display: 'flex', flexDirection: 'column', gap: tokens.spacingVerticalS }, + row: { display: 'flex', flexWrap: 'wrap', alignItems: 'center', gap: tokens.spacingHorizontalS }, + action: { ...mobileTouchTargetHeight }, +}) diff --git a/frontend/src/components/Parameters/ReferenceField.tsx b/frontend/src/components/Parameters/ReferenceField.tsx new file mode 100644 index 0000000000..dbc50a2fcc --- /dev/null +++ b/frontend/src/components/Parameters/ReferenceField.tsx @@ -0,0 +1,83 @@ +import { useId, useState } from 'react' + +import { Button, Field, Select, Text } from '@fluentui/react-components' + +import type { RegistryReferenceOption } from '@/types' + +import { useReferenceFieldStyles } from './ReferenceField.styles' + +interface ReferenceFieldProps { + label: string + options: RegistryReferenceOption[] + value: string | string[] + multiple?: boolean + disabled?: boolean + hint?: string + onChange: (value: string | string[]) => void +} + +/** Ordered registry references. Repeated entries are intentional. */ +export default function ReferenceField({ + label, options, value, multiple = false, disabled = false, hint, onChange, +}: ReferenceFieldProps) { + const styles = useReferenceFieldStyles() + const [pending, setPending] = useState('') + const [entryIds, setEntryIds] = useState([]) + const id = useId() + const [nextId, setNextId] = useState(0) + const selected = Array.isArray(value) ? value : [] + + const move = (index: number, delta: number): void => { + const next = [...selected] + const ids = selected.map((_, position) => entryIds[position] ?? `${id}-initial-${position}`) + ;[next[index], next[index + delta]] = [next[index + delta], next[index]] + ;[ids[index], ids[index + delta]] = [ids[index + delta], ids[index]] + setEntryIds(ids) + onChange(next) + } + + return ( +
+ + + + {multiple && ( + <> + + {selected.map((name, index) => ( +
+ {index + 1}. {name} + + + +
+ ))} + + )} +
+ ) +} diff --git a/frontend/src/components/Registry/CreateTechniqueDialog.styles.ts b/frontend/src/components/Registry/CreateTechniqueDialog.styles.ts new file mode 100644 index 0000000000..5f2a525097 --- /dev/null +++ b/frontend/src/components/Registry/CreateTechniqueDialog.styles.ts @@ -0,0 +1,19 @@ +import { makeStyles, tokens } from '@fluentui/react-components' + +import { mobileTouchTargetHeight, NARROW_VIEWPORT_QUERY } from '@/styles/touchTargets' + +export const useCreateTechniqueDialogStyles = makeStyles({ + surface: { + width: '100%', minWidth: 0, maxWidth: '37.5rem', maxHeight: '90vh', + [NARROW_VIEWPORT_QUERY]: { + maxWidth: `calc(100vw - ${tokens.spacingHorizontalXXL} - ${tokens.spacingHorizontalXXL})`, + }, + }, + content: { minWidth: 0, overflowY: 'auto', maxHeight: '65vh' }, + form: { + display: 'flex', flexDirection: 'column', width: '100%', minWidth: 0, + maxWidth: '100%', gap: tokens.spacingVerticalL, + }, + select: { width: '100%', minWidth: 0, '& select': { width: '100%', minWidth: 0 } }, + action: { ...mobileTouchTargetHeight }, +}) diff --git a/frontend/src/components/Registry/CreateTechniqueDialog.tsx b/frontend/src/components/Registry/CreateTechniqueDialog.tsx new file mode 100644 index 0000000000..01e2509a64 --- /dev/null +++ b/frontend/src/components/Registry/CreateTechniqueDialog.tsx @@ -0,0 +1,217 @@ +import { useEffect, useRef, useState } from 'react' + +import { + Button, Dialog, DialogActions, DialogBody, DialogContent, DialogSurface, DialogTitle, + Field, Input, MessageBar, MessageBarBody, Select, Spinner, Textarea, +} from '@fluentui/react-components' + +import ParameterField from '@/components/Parameters/ParameterField' +import ReferenceField from '@/components/Parameters/ReferenceField' +import { buildParametersFromForm, getInitialFormValues, type ParameterFormValue } from '@/components/Parameters/parameterForm' +import { convertersApi, targetsApi, techniquesApi } from '@/services/api' +import { toApiError } from '@/services/errors' +import type { CreateTechniqueRequest, Parameter, RegistryReferenceOption, TechniqueTypeEntry } from '@/types' +import { fetchAllPages } from '@/utils/fetchAllPages' + +import { useCreateTechniqueDialogStyles } from './CreateTechniqueDialog.styles' + +const SCALAR_TYPES = new Set(['str', 'int', 'float', 'bool', 'list[str]', 'list[int]', 'list[float]', 'list[bool]']) + +function canConfigure(parameter: Parameter, allowStructured = true): boolean { + if (parameter.reference_type) return false + if (parameter.variants) { + return allowStructured && !parameter.is_list && Object.values(parameter.variants).some((parameters) => + parameters.every((nested) => !nested.required || canConfigure(nested, false))) + } + return SCALAR_TYPES.has(parameter.type_name) || Boolean(parameter.choices?.length) +} + +function guiParameters(parameters: Parameter[], allowStructured = true): Parameter[] { + return parameters.filter((parameter) => canConfigure(parameter, allowStructured)).map((parameter) => parameter.variants ? { + ...parameter, + variants: Object.fromEntries(Object.entries(parameter.variants) + .filter(([, nested]) => nested.every((entry) => !entry.required || canConfigure(entry, false))) + .map(([name, nested]) => [name, guiParameters(nested, false)])), + } : parameter) +} + +function getAttackParameters(type?: TechniqueTypeEntry): Parameter[] { + return type?.parameters.filter((parameter) => parameter.name !== 'attack_converter_config') ?? [] +} + +interface CreateTechniqueDialogProps { + onClose: () => void + onCreated: () => void +} + +/** Configure existing attacks with registry references. Advanced factories use Python. */ +export default function CreateTechniqueDialog({ onClose, onCreated }: CreateTechniqueDialogProps) { + const styles = useCreateTechniqueDialogStyles() + const [types, setTypes] = useState([]) + const [targetOptions, setTargetOptions] = useState([]) + const [converterOptions, setConverterOptions] = useState([]) + const [selectedType, setSelectedType] = useState('') + const [name, setName] = useState('') + const [description, setDescription] = useState('') + const [tags, setTags] = useState('') + const [values, setValues] = useState>({}) + const [requestConverters, setRequestConverters] = useState([]) + const [responseConverters, setResponseConverters] = useState([]) + const [adversarialTarget, setAdversarialTarget] = useState('') + const [systemPrompt, setSystemPrompt] = useState('') + const [seedPrompt, setSeedPrompt] = useState('') + const [turnPrompt, setTurnPrompt] = useState('') + const [loading, setLoading] = useState(true) + const [submitting, setSubmitting] = useState(false) + const [error, setError] = useState(null) + const [retry, setRetry] = useState(0) + const active = useRef(true) + const errorRef = useRef(null) + const [submitError, setSubmitError] = useState(false) + + useEffect(() => { + active.current = true + return () => { active.current = false } + }, []) + + useEffect(() => { + let ignore = false + void Promise.resolve().then(async () => { + if (ignore) return + setLoading(true) + setError(null) + try { + const [metadata, targets, converters] = await Promise.all([ + techniquesApi.listTypes(), + fetchAllPages( + (cursor: string | undefined) => targetsApi.listTargets(200, cursor), + undefined, + (entry) => entry.target_registry_name, + true, + ), + convertersApi.listConverters(), + ]) + if (ignore) return + setTypes(metadata.items.filter((entry) => + getAttackParameters(entry).every((parameter) => !parameter.required || canConfigure(parameter)))) + setTargetOptions(targets.map((entry) => ({ name: entry.target_registry_name, type: entry.identifier.class_name }))) + setConverterOptions(converters.items.map((entry) => ({ name: entry.converter_id, type: entry.identifier.class_name }))) + } catch (err) { + if (!ignore) setError(toApiError(err).detail) + } finally { + if (!ignore) setLoading(false) + } + }) + return () => { ignore = true } + }, [retry]) + + useEffect(() => { + if (submitError && error) errorRef.current?.focus() + }, [error, submitError]) + + const selected = types.find((entry) => entry.attack_type === selectedType) + const parameters = guiParameters(getAttackParameters(selected)) + + const submit = async (): Promise => { + const fail = (message: string): void => { setError(message); setSubmitError(true) } + const parsedTags = tags.split(',').map((tag) => tag.trim()).filter(Boolean) + if (!selected) { + fail('Select an attack type.') + return + } + const result = buildParametersFromForm(parameters, values) + if (!result.ok) { fail(result.error); return } + const request: CreateTechniqueRequest = { + name, description, tags: parsedTags, type: selectedType, + params: result.parameters ?? {}, + } + if (selected.supports_converters && (requestConverters.length || responseConverters.length)) { + request.request_converters = requestConverters + request.response_converters = responseConverters + } + if (selected.supports_adversarial) { + if (adversarialTarget) request.adversarial_chat = adversarialTarget + if (systemPrompt) request.adversarial_system_prompt = systemPrompt + if (seedPrompt) request.adversarial_seed_prompt = seedPrompt + if (turnPrompt) request.adversarial_prompt_template = turnPrompt + } + setSubmitting(true) + setError(null) + setSubmitError(false) + try { + await techniquesApi.createTechnique(request) + if (active.current) onCreated() + } catch (err) { + if (active.current) fail(toApiError(err).detail) + } finally { + if (active.current) setSubmitting(false) + } + } + + return ( + { if (!data.open) onClose() }}> + + + New technique + +
+ {loading && } + {error && {error}} + {!loading && types.length === 0 && } + {!loading && types.length > 0 && ( + <> + + setName(data.value)} /> + +