Skip to content

FIX Recover chat drafts after target processing failures #2474

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Roman Lutz (romanlutz) wants to merge 5 commits into
base: main
Choose a base branch
from
+1,549 −96
Open

FIX Recover chat drafts after target processing failures #2474

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b7f7fc8
Fix chat recovery after target processing errors
Copilot Aug 21, 2026
3c62342
Polish chat recovery presentation
romanlutz Aug 24, 2026
2656516
Stabilize chat message E2E assertions
romanlutz Aug 24, 2026
c90ef65
Merge branch 'main' into romanlutz-fix-chat-target-recovery
romanlutz Aug 24, 2026
b2d3f55
Address chat recovery review feedback
romanlutz Aug 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 26 additions & 22 deletions frontend/e2e/chat.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { readFileSync } from "node:fs";
import { test, expect, type Page } from "@playwright/test";
import { test, expect, type Locator, type Page } from "@playwright/test";
import { makeTarget } from "./_targets";

// ---------------------------------------------------------------------------
Expand All @@ -10,6 +10,10 @@ const MOCK_CONVERSATION_ID = "e2e-conv-001";
const WIDE_IMAGE_DATA_URI =
"data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='800' height='600' viewBox='0 0 800 600'%3E%3Crect width='800' height='600' fill='%230078d4'/%3E%3C/svg%3E";

function getMessageByText(page: Page, text: string): Locator {
return page.getByTestId("message-list").getByText(text, { exact: true });
}

/** Intercept targets & attacks APIs so the chat flow can run without real keys. */
async function mockBackendAPIs(page: Page) {
// Accumulate messages so multi-turn tests get full history back
Expand Down Expand Up @@ -236,7 +240,7 @@ test.describe("Chat Functionality", () => {
const input = page.getByRole("textbox");
await input.fill("Start a mobile conversation");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("Start a mobile conversation", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Start a mobile conversation")).toBeVisible();

await page.setViewportSize({ width: 390, height: 844 });
const chatArea = page.getByTestId("chat-area");
Expand Down Expand Up @@ -277,7 +281,7 @@ test.describe("Chat Functionality", () => {
await page.getByRole("button", { name: /send/i }).click();

// User message appears
await expect(page.getByText("Hello, this is a test message", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Hello, this is a test message")).toBeVisible();

// Backend response appears
await expect(
Expand Down Expand Up @@ -313,7 +317,7 @@ test.describe("Chat Functionality", () => {
await input.fill("First message");
await page.getByRole("button", { name: /send/i }).click();

await expect(page.getByText("First message", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "First message")).toBeVisible();
await expect(
page.getByText("Mock response for: First message"),
).toBeVisible({ timeout: 10000 });
Expand All @@ -339,22 +343,22 @@ test.describe("Multiple Messages", () => {
// Send first message
await input.fill("First message");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("First message", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "First message")).toBeVisible();
await expect(
page.getByText("Mock response for: First message"),
).toBeVisible({ timeout: 10000 });

// Send second message
await input.fill("Second message");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("Second message", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Second message")).toBeVisible();
await expect(
page.getByText("Mock response for: Second message"),
).toBeVisible({ timeout: 10000 });

// Both user messages should still be visible
await expect(page.getByText("First message", { exact: true })).toBeVisible();
await expect(page.getByText("Second message", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "First message")).toBeVisible();
await expect(getMessageByText(page, "Second message")).toBeVisible();
});
});

Expand Down Expand Up @@ -505,7 +509,7 @@ test.describe("Multi-modal: Image response", () => {
await page.getByRole("button", { name: /send/i }).click();

// User message visible
await expect(page.getByText("Generate an image", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Generate an image")).toBeVisible();

// Image element should appear (exclude logo)
const img = page.locator('img:not([alt="Co-PyRIT Logo"])');
Expand Down Expand Up @@ -603,7 +607,7 @@ test.describe("Multi-modal: Audio response", () => {
await input.fill("Speak this out loud");
await page.getByRole("button", { name: /send/i }).click();

await expect(page.getByText("Speak this out loud", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Speak this out loud")).toBeVisible();

// Audio element should appear
const audio = page.locator("audio");
Expand Down Expand Up @@ -666,7 +670,7 @@ test.describe("Multi-modal: Video response", () => {
await input.fill("Create a video clip");
await page.getByRole("button", { name: /send/i }).click();

await expect(page.getByText("Create a video clip", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Create a video clip")).toBeVisible();

// Video element should appear
const video = page.locator("video");
Expand Down Expand Up @@ -707,7 +711,7 @@ test.describe("Multi-modal: Mixed text + image response", () => {
await page.getByRole("button", { name: /send/i }).click();

// Both text and image should be visible
await expect(page.getByText("Here is the analysis:", { exact: true })).toBeVisible({ timeout: 10000 });
await expect(getMessageByText(page, "Here is the analysis:")).toBeVisible({ timeout: 10000 });
const img = page.locator('img:not([alt="Co-PyRIT Logo"])');
await expect(img).toBeVisible({ timeout: 10000 });
});
Expand Down Expand Up @@ -736,7 +740,7 @@ test.describe("Multi-modal: Error response from target", () => {
await input.fill("unsafe prompt");
await page.getByRole("button", { name: /send/i }).click();

await expect(page.getByText("unsafe prompt", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "unsafe prompt")).toBeVisible();

// Error should be displayed
await expect(
Expand All @@ -758,31 +762,31 @@ test.describe("Multi-turn conversation flow", () => {
// Turn 1
await input.fill("First turn");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("First turn", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "First turn")).toBeVisible();
await expect(
page.getByText("Mock response for: First turn"),
).toBeVisible({ timeout: 10000 });

// Turn 2
await input.fill("Second turn");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("Second turn", { exact: true })).toBeVisible({ timeout: 10000 });
await expect(getMessageByText(page, "Second turn")).toBeVisible({ timeout: 10000 });
await expect(
page.getByText("Mock response for: Second turn"),
).toBeVisible({ timeout: 10000 });

// Turn 3
await input.fill("Third turn");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("Third turn", { exact: true })).toBeVisible({ timeout: 10000 });
await expect(getMessageByText(page, "Third turn")).toBeVisible({ timeout: 10000 });
await expect(
page.getByText("Mock response for: Third turn"),
).toBeVisible({ timeout: 10000 });

// All previous messages still visible
await expect(page.getByText("First turn", { exact: true })).toBeVisible();
await expect(page.getByText("Second turn", { exact: true })).toBeVisible();
await expect(page.getByText("Third turn", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "First turn")).toBeVisible();
await expect(getMessageByText(page, "Second turn")).toBeVisible();
await expect(getMessageByText(page, "Third turn")).toBeVisible();
});

test("should reset conversation on New Chat and send again", async ({ page }) => {
Expand All @@ -791,19 +795,19 @@ test.describe("Multi-turn conversation flow", () => {
// Send a message
await input.fill("Before reset");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("Before reset", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "Before reset")).toBeVisible();
await expect(
page.getByText("Mock response for: Before reset"),
).toBeVisible({ timeout: 10000 });

// New Attack
await page.getByTestId("new-attack-btn").click();
await expect(page.getByText("Before reset", { exact: true })).not.toBeVisible();
await expect(getMessageByText(page, "Before reset")).not.toBeVisible();

// Send new message in fresh conversation
await input.fill("After reset");
await page.getByRole("button", { name: /send/i }).click();
await expect(page.getByText("After reset", { exact: true })).toBeVisible();
await expect(getMessageByText(page, "After reset")).toBeVisible();
await expect(
page.getByText("Mock response for: After reset"),
).toBeVisible({ timeout: 10000 });
Expand Down
145 changes: 144 additions & 1 deletion frontend/e2e/errors.spec.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ function buildSuccessMessageMock(userText: string) {
messages: {
messages: [
{
turn_number: 1,
turn_number: 0,
role: "user",
created_at: new Date().toISOString(),
message_pieces: [
Expand Down Expand Up @@ -49,6 +49,48 @@ function buildSuccessMessageMock(userText: string) {
};
}

function buildProcessingFailureMock(userText: string) {
return {
messages: {
messages: [
{
turn_number: 2,
role: "user",
created_at: new Date().toISOString(),
message_pieces: [
{
id: "p-processing-user",
original_value_data_type: "text",
converted_value_data_type: "text",
original_value: userText,
converted_value: userText,
scores: [],
response_error: "none",
},
],
},
{
turn_number: 3,
role: "assistant",
created_at: new Date().toISOString(),
message_pieces: [
{
id: "p-processing-error",
original_value_data_type: "text",
converted_value_data_type: "error",
original_value: "",
converted_value:
"RuntimeError: target failed\nTraceback (most recent call last): ...",
scores: [],
response_error: "processing",
},
],
},
],
},
};
}

/**
* Set up all the mocks needed for a full chat flow.
*
Expand Down Expand Up @@ -206,6 +248,107 @@ async function triggerVisibilityChange(page: Page) {
});
}

// ---------------------------------------------------------------------------
// Error scenario: persisted target processing failure returned with HTTP 200
// ---------------------------------------------------------------------------

test.describe("Error: target processing failure returned with HTTP 200", () => {
test("should preserve the draft and offer edit recovery", async ({ page }) => {
let callCount = 0;
let recoveryRequest: Record<string, unknown> | undefined;
let recoveryCreated = false;
await mockAllAPIs(page, async (route) => {
const body = JSON.parse(route.request().postData() ?? "{}");
const userText =
body?.pieces?.find(
(piece: Record<string, string>) => piece.data_type === "text",
)?.original_value || "message";
callCount++;
const response = callCount === 1
? buildSuccessMessageMock(userText)
: buildProcessingFailureMock(userText);
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify(response),
});
});
await page.route(/\/api\/attacks\/[^/]+\/conversations/, async (route) => {
if (route.request().method() === "GET" && recoveryCreated) {
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({
attack_result_id: "err-ar-001",
main_conversation_id: MOCK_CONV_ID,
conversations: [
{
conversation_id: MOCK_CONV_ID,
message_count: 4,
last_message_preview: "Target response error",
created_at: "2026-01-01T00:00:00.000Z",
},
{
conversation_id: "err-conv-recovery",
message_count: 2,
last_message_preview: "Reply to: Setup message",
created_at: "2026-01-01T00:00:01.000Z",
},
],
}),
});
return;
}
if (route.request().method() !== "POST") {
await route.fallback();
return;
}
recoveryRequest = JSON.parse(route.request().postData() ?? "{}");
recoveryCreated = true;
await route.fulfill({
status: 200,
contentType: "application/json",
body: JSON.stringify({
conversation_id: "err-conv-recovery",
created_at: "2026-01-01T00:00:01.000Z",
}),
});
});

await page.goto("/");
await activateMockTarget(page);
await sendAndWait(page, "Setup message", "Reply to: Setup message");

const input = page.getByRole("textbox");
await input.fill("Preserve this draft");
await page.getByRole("button", { name: /send/i }).click();

await expect(
page.getByText(/target could not process this message/i),
).toBeVisible();
const recoveryButton = page.getByRole("button", {
name: /edit in clean conversation/i,
});
await expect(recoveryButton).toHaveCount(1);
await expect(input).toHaveValue("Preserve this draft");
await expect(input).toBeDisabled();
await expect(page.getByText(/Traceback \(most recent call last\)/i)).toHaveCount(0);

await recoveryButton.click();
await expect.poll(() => recoveryRequest).toEqual({
source_conversation_id: MOCK_CONV_ID,
cutoff_index: 1,
});
await expect(input).toBeFocused();
await expect(input).toHaveValue("Preserve this draft");
await expect(page.getByTestId(`conversation-item-${MOCK_CONV_ID}`)).toBeVisible();
await expect(
page.getByTestId("conversation-item-err-conv-recovery"),
).toBeVisible();
expect(callCount).toBe(2);
});
});

// ---------------------------------------------------------------------------
// Error scenario: backend returns 500 on send message
// ---------------------------------------------------------------------------
Expand Down
Loading