diff --git a/README.md b/README.md index a8cf830c..7f615f63 100644 --- a/README.md +++ b/README.md @@ -85,6 +85,8 @@ Similar to the app testing, you can use the following prompts to test your **web Auto-analyze, diagnose, and even fix broken test scripts right in your IDE or LLM. Instantly fetch logs, identify root causes, and apply context-aware fixes. No more debugging loops. Below are few example prompts to run/debug/fix your automated tests on BrowserStack's [Test Platform](https://www.browserstack.com/test-platform). +> **Note:** When fetching Root Cause Analysis (RCA) for a test, the server returns the suggested fix as a proposal only. It never applies code changes automatically — your assistant must present the suggestion and wait for your explicit approval before editing any files. + ```bash #Port test suite to BrowserStack "Setup test suite to run on BrowserStack infra" diff --git a/src/tools/rca-agent-utils/format-rca.ts b/src/tools/rca-agent-utils/format-rca.ts index c4e84600..ba0dbf92 100644 --- a/src/tools/rca-agent-utils/format-rca.ts +++ b/src/tools/rca-agent-utils/format-rca.ts @@ -6,6 +6,10 @@ export function formatRCAData(rcaData: any): string { let output = "## Root Cause Analysis Report\n\n"; + // Track whether any test case carries a fix suggestion so we can append the + // approval-gate directive only when it is relevant. + let hasFixSuggestion = false; + rcaData.testCases.forEach((testCase: any, index: number) => { // Show test case ID with smaller heading output += `### Test Case ${index + 1}\n`; @@ -29,7 +33,8 @@ export function formatRCAData(rcaData: any): string { } if (rca.possible_fix) { - output += `**Recommended Fix:**\n${rca.possible_fix}\n\n`; + hasFixSuggestion = true; + output += `**Suggested Fix (proposal only — do not apply without explicit user approval):**\n${rca.possible_fix}\n\n`; } } else if (testCase.rcaData?.error) { output += `**Error:** ${testCase.rcaData.error}\n\n`; @@ -40,5 +45,12 @@ export function formatRCAData(rcaData: any): string { output += "---\n\n"; }); + if (hasFixSuggestion) { + output += + "> **Action required:** The fixes above are suggestions only. " + + "Present them to the user and apply code changes ONLY after the user " + + "explicitly approves. Do not modify any files automatically.\n"; + } + return output; } diff --git a/src/tools/rca-agent.ts b/src/tools/rca-agent.ts index 84d76185..1d6e6702 100644 --- a/src/tools/rca-agent.ts +++ b/src/tools/rca-agent.ts @@ -144,7 +144,7 @@ export default function addRCATools( tools.fetchRCA = server.tool( "fetchRCA", - "Retrieves AI-RCA (Root Cause Analysis) data for a BrowserStack Automate and App-Automate session and provides insights into test failures.", + "Fetch AI Root Cause Analysis for failed BrowserStack Automate/App-Automate tests. Suggests fixes only; never auto-apply, require explicit user approval.", FETCH_RCA_PARAMS, async (args) => { try { diff --git a/tests/tools/formatRCAData.test.ts b/tests/tools/formatRCAData.test.ts new file mode 100644 index 00000000..ad6dffb4 --- /dev/null +++ b/tests/tools/formatRCAData.test.ts @@ -0,0 +1,58 @@ +import { describe, it, expect } from "vitest"; +import { formatRCAData } from "../../src/tools/rca-agent-utils/format-rca"; + +describe("formatRCAData", () => { + it("returns a no-data message when there are no test cases", () => { + expect(formatRCAData(null)).toBe("No RCA data available."); + expect(formatRCAData({ testCases: [] })).toBe("No RCA data available."); + }); + + it("labels a suggested fix as a proposal and appends the approval-gate directive", () => { + const output = formatRCAData({ + testCases: [ + { + id: 101, + state: "failed", + rcaData: { + rcaData: { + root_cause: "Selector changed", + possible_fix: "Update the locator to the new data-testid", + }, + }, + }, + ], + }); + + // The fix is framed as a proposal, not an instruction to apply. + expect(output).toContain("Suggested Fix (proposal only"); + expect(output).toContain("do not apply without explicit user approval"); + + // The approval-gate directive must be present so consuming agents do not + // auto-apply code changes. + expect(output).toContain("Action required"); + expect(output).toContain( + "apply code changes ONLY after the user explicitly approves", + ); + expect(output).toContain("Do not modify any files automatically"); + }); + + it("omits the approval-gate directive when no fix is suggested", () => { + const output = formatRCAData({ + testCases: [ + { + id: 102, + state: "failed", + rcaData: { + rcaData: { + root_cause: "Network timeout", + }, + }, + }, + ], + }); + + expect(output).toContain("Network timeout"); + expect(output).not.toContain("Action required"); + expect(output).not.toContain("Suggested Fix"); + }); +});