diff --git a/README.md b/README.md index fcd915e..2eb3899 100644 --- a/README.md +++ b/README.md @@ -154,8 +154,8 @@ artifact에는 다음 파일이 포함됩니다. | `branch-pairs.json` | base branch와 감시 대상 branch 전체의 이름순 비교 조합 | | `deterministic-evidence.json` | git merge signal, changed files, changed hunks, check/PR metadata, deterministic risk 결과 | | `ai-target-selection.json` | AI 호출 대상 branch와 skipped branch 사유 | -| `ai-prompt.json` | OpenAI에 전달한 system prompt, user prompt, response shape | -| `ai-response.json` | provider가 반환한 raw response | +| `ai-prompt.json` | OpenAI 요청 대상 branch, system prompt, response shape, 코드 원문을 제외한 file·line range·길이·hash·잘림 여부 | +| `ai-response.json` | provider response와 제안 patch 원문을 제외한 byte length·line count·hunk range | | `ai-error.json` | provider 호출 또는 response validation 실패 요약. 실패가 없으면 생성되지 않을 수 있음 | | `ai-result.json` | response validation 이후 branch별 AI prediction, skipped, failed 매핑 결과 | | `report.md` | 최종 Markdown report | diff --git a/src/debug/aiPredictionArtifact.ts b/src/debug/aiPredictionArtifact.ts new file mode 100644 index 0000000..2eb1689 --- /dev/null +++ b/src/debug/aiPredictionArtifact.ts @@ -0,0 +1,203 @@ +import { createHash } from "node:crypto" + +type AiPredictionPromptDebugEventInput = { + targetBranches: Array<{ + branchName: string + baseBranch: string + }> + prompt: { + systemPrompt: string + userPrompt: string + responseShape?: string + } +} + +type AiPredictionResponseDebugEventInput = { + targetBranches: Array<{ + branchName: string + baseBranch: string + }> + response: unknown +} + +export type AiPredictionPromptDebugArtifact = { + targetBranches: Array<{ + branchName: string + baseBranch: string + }> + prompt: { + systemPrompt: string + userPrompt: string + responseShape?: string + } +} + +export type AiPredictionResponseDebugArtifact = { + targetBranches: Array<{ + branchName: string + baseBranch: string + }> + response: unknown +} + +// OpenAI에 전달된 prompt event에서 consumer repository 코드 원문만 metadata로 치환 +export function sanitizeAiPredictionPromptDebugEvent( + event: AiPredictionPromptDebugEventInput +): AiPredictionPromptDebugArtifact { + return { + targetBranches: event.targetBranches.map(target => ({ + branchName: target.branchName, + baseBranch: target.baseBranch + })), + prompt: { + systemPrompt: event.prompt.systemPrompt, + userPrompt: sanitizedUserPromptFor(event.prompt.userPrompt), + ...(event.prompt.responseShape + ? { responseShape: event.prompt.responseShape } + : {}) + } + } +} + +// provider response에서 제안 patch 원문만 크기와 hunk metadata로 치환 +export function sanitizeAiPredictionResponseDebugEvent( + event: AiPredictionResponseDebugEventInput +): AiPredictionResponseDebugArtifact { + return { + targetBranches: event.targetBranches.map(target => ({ + branchName: target.branchName, + baseBranch: target.baseBranch + })), + response: sanitizedResponseValueFor(event.response) + } +} + +// JSON prompt는 구조를 유지하고 비정형 prompt는 원문 없이 크기와 hash만 기록 +function sanitizedUserPromptFor(userPrompt: string): string { + try { + return JSON.stringify(sanitizedJsonValueFor(JSON.parse(userPrompt)), null, 2) + } catch { + return JSON.stringify(contentMetadataFor(userPrompt, true), null, 2) + } +} + +// 중첩된 AI evidence에서 content 문자열을 제거하고 추적용 metadata를 추가 +function sanitizedJsonValueFor(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sanitizedJsonValueFor) + } + + if (!isJsonObject(value)) { + return value + } + + const sanitized = Object.fromEntries(Object.entries(value) + .filter(([name, item]) => name !== "content" || typeof item !== "string") + .map(([name, item]) => [name, sanitizedJsonValueFor(item)])) + const content = typeof value.content === "string" ? value.content : undefined + + if (content === undefined) { + return sanitized + } + + const startLine = typeof value.startLine === "number" ? value.startLine : undefined + const endLine = typeof value.endLine === "number" ? value.endLine : undefined + + return { + ...sanitized, + ...contentMetadataFor(content), + ...(startLine !== undefined && endLine !== undefined + ? { lineCount: Math.max(endLine - startLine + 1, 0) } + : {}) + } +} + +// 코드 원문을 재현하지 않고 동일 문맥 여부와 크기만 비교할 metadata 구성 +function contentMetadataFor(content: string, redacted = false): { + contentByteLength: number + contentHash: string + redacted?: true +} { + return { + contentByteLength: Buffer.byteLength(content, "utf8"), + contentHash: createHash("sha256").update(content, "utf8").digest("hex"), + ...(redacted ? { redacted: true as const } : {}) + } +} + +// 중첩된 provider response에서 patch 문자열만 metadata로 변환 +function sanitizedResponseValueFor(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map(sanitizedResponseValueFor) + } + + if (!isJsonObject(value)) { + return value + } + + return Object.fromEntries(Object.entries(value).map(([name, item]) => [ + name, + name === "patch" && typeof item === "string" + ? patchMetadataFor(item) + : sanitizedResponseValueFor(item) + ])) +} + +// unified diff 원문을 저장하지 않고 크기와 변경 범위만 추적할 metadata 구성 +function patchMetadataFor(patch: string): { + byteLength: number + lineCount: number + hunkRanges: Array<{ + oldStart: number + oldCount: number + newStart: number + newCount: number + }> +} { + return { + byteLength: Buffer.byteLength(patch, "utf8"), + lineCount: lineCountFor(patch), + hunkRanges: hunkRangesFor(patch) + } +} + +// 마지막 줄바꿈은 별도 빈 줄로 세지 않고 patch 줄 수를 계산 +function lineCountFor(value: string): number { + if (value.length === 0) { + return 0 + } + + const lines = value.split(/\r\n|\n|\r/) + return lines[lines.length - 1] === "" ? lines.length - 1 : lines.length +} + +// unified diff hunk header의 기존/변경 line 범위를 입력 순서대로 추출 +function hunkRangesFor(patch: string): Array<{ + oldStart: number + oldCount: number + newStart: number + newCount: number +}> { + const ranges: Array<{ + oldStart: number + oldCount: number + newStart: number + newCount: number + }> = [] + const pattern = /^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))? @@/gm + + for (const match of patch.matchAll(pattern)) { + ranges.push({ + oldStart: Number(match[1]), + oldCount: Number(match[2] ?? "1"), + newStart: Number(match[3]), + newCount: Number(match[4] ?? "1") + }) + } + + return ranges +} + +function isJsonObject(value: unknown): value is Record { + return typeof value === "object" && value !== null +} diff --git a/src/workflows/mergeRiskWatch.ts b/src/workflows/mergeRiskWatch.ts index a98e5cb..f215b23 100644 --- a/src/workflows/mergeRiskWatch.ts +++ b/src/workflows/mergeRiskWatch.ts @@ -19,6 +19,10 @@ import { select as selectAiPredictionTargets } from "../ai/predictionTargetSelec import { build as buildMergeRiskReport } from "../reports/reportBuilder.js" import { format as formatMergeRiskReportMarkdown } from "../reports/markdownFormatter.js" import { send as sendMergeRiskReport } from "../reportChannels/reportChannel.js" +import { + sanitizeAiPredictionPromptDebugEvent, + sanitizeAiPredictionResponseDebugEvent +} from "../debug/aiPredictionArtifact.js" import { writerFor as debugArtifactWriterFor } from "../debug/debugArtifact.js" import type { AiPredictionEvidencePayload @@ -216,10 +220,16 @@ export async function run(options: MergeRiskWatchOptions): Promise { { debugObserver: debugArtifactWriter ? { - // 생성된 AI prompt를 debug artifact로 기록 - onPromptBuilt: event => debugArtifactWriter.writeJson("ai-prompt.json", event), - // 받은 AI response를 debug artifact로 기록 - onResponseReceived: event => debugArtifactWriter.writeJson("ai-response.json", event), + // 생성된 AI prompt에서 코드 원문을 제거한 표현만 debug artifact로 기록 + onPromptBuilt: event => debugArtifactWriter.writeJson( + "ai-prompt.json", + sanitizeAiPredictionPromptDebugEvent(event) + ), + // 받은 AI response에서 제안 patch 원문을 제거한 표현만 debug artifact로 기록 + onResponseReceived: event => debugArtifactWriter.writeJson( + "ai-response.json", + sanitizeAiPredictionResponseDebugEvent(event) + ), // AI prediction 실패 정보를 debug artifact로 기록 onPredictionFailed: event => debugArtifactWriter.writeJson("ai-error.json", event) } diff --git a/tests/debug/aiPredictionArtifact.test.ts b/tests/debug/aiPredictionArtifact.test.ts new file mode 100644 index 0000000..f9730fd --- /dev/null +++ b/tests/debug/aiPredictionArtifact.test.ts @@ -0,0 +1,165 @@ +import test from "node:test" +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { + sanitizeAiPredictionPromptDebugEvent, + sanitizeAiPredictionResponseDebugEvent +} from "../../src/debug/aiPredictionArtifact.js" + +// AI prompt artifact가 코드 원문 대신 file과 line metadata를 기록하는지 확인 +test("replaces AI prompt code context with metadata", () => { + const content = "const consumerSourceMarker = true\nreturn consumerSourceMarker\n" + const event = { + targetBranches: [{ + branchName: "feature/left", + baseBranch: "main" + }], + prompt: { + systemPrompt: "Review merge risk", + userPrompt: JSON.stringify({ + codeContext: { + evidence: [{ + leftSnippet: { + status: "text", + filePath: "Sources/Feature.swift", + content, + startLine: 12, + endLine: 13, + truncated: false + } + }] + } + }, null, 2), + responseShape: "predictionPairCleanOverlap" + } + } + + const artifact = sanitizeAiPredictionPromptDebugEvent(event) + const payload = JSON.parse(artifact.prompt.userPrompt) as { + codeContext?: { + evidence?: Array<{ + leftSnippet?: Record + }> + } + } + + assert.deepEqual(payload.codeContext?.evidence?.[0]?.leftSnippet, { + status: "text", + filePath: "Sources/Feature.swift", + startLine: 12, + endLine: 13, + truncated: false, + contentByteLength: Buffer.byteLength(content, "utf8"), + contentHash: createHash("sha256").update(content, "utf8").digest("hex"), + lineCount: 2 + }) + assert.equal(JSON.parse(event.prompt.userPrompt).codeContext.evidence[0].leftSnippet.content, content) + assert.doesNotMatch(JSON.stringify(artifact), /consumerSourceMarker/) +}) + +// 빈 코드 문맥도 0 byte 원문 대신 line과 hash metadata를 기록하는지 확인 +test("records metadata for empty AI prompt code context", () => { + const artifact = sanitizeAiPredictionPromptDebugEvent({ + targetBranches: [], + prompt: { + systemPrompt: "Review merge risk", + userPrompt: JSON.stringify({ + snippet: { + filePath: "Sources/Empty.swift", + content: "", + startLine: 1, + endLine: 1, + truncated: false + } + }) + } + }) + const payload = JSON.parse(artifact.prompt.userPrompt) as { + snippet?: Record + } + + assert.deepEqual(payload.snippet, { + filePath: "Sources/Empty.swift", + startLine: 1, + endLine: 1, + truncated: false, + contentByteLength: 0, + contentHash: createHash("sha256").update("", "utf8").digest("hex"), + lineCount: 1 + }) +}) + +// 비정형 user prompt도 원문을 저장하지 않고 크기와 hash만 기록하는지 확인 +test("redacts non-JSON AI user prompt", () => { + const userPrompt = "consumer repository source" + const event = { + targetBranches: [], + prompt: { + systemPrompt: "Review merge risk", + userPrompt + } + } + + const artifact = sanitizeAiPredictionPromptDebugEvent(event) + + assert.deepEqual(JSON.parse(artifact.prompt.userPrompt), { + contentByteLength: Buffer.byteLength(userPrompt, "utf8"), + contentHash: createHash("sha256").update(userPrompt, "utf8").digest("hex"), + redacted: true + }) + assert.equal(event.prompt.userPrompt, userPrompt) + assert.doesNotMatch(JSON.stringify(artifact), /consumer repository source/) +}) + +// AI response artifact가 제안 patch 원문 대신 크기와 hunk 범위를 기록하는지 확인 +test("replaces AI response patch with metadata", () => { + const patch = [ + "--- a/Sources/Feature.swift", + "+++ b/Sources/Feature.swift", + "@@ -10,2 +10,2 @@", + "-let consumerSourceMarker = false", + "+let consumerSourceMarker = true", + "@@ -30 +30,3 @@ apply", + "-return value", + "+return value", + "+return result" + ].join("\n") + const artifact = sanitizeAiPredictionResponseDebugEvent({ + targetBranches: [{ + branchName: "feature/left", + baseBranch: "main" + }], + response: { + kind: "confirmed_conflict", + patches: [{ + filePath: "Sources/Feature.swift", + patch, + reason: "충돌 상태 갱신" + }] + } + }) + const response = artifact.response as { + patches?: Array> + } + + assert.deepEqual(response.patches?.[0], { + filePath: "Sources/Feature.swift", + patch: { + byteLength: Buffer.byteLength(patch, "utf8"), + lineCount: 9, + hunkRanges: [{ + oldStart: 10, + oldCount: 2, + newStart: 10, + newCount: 2 + }, { + oldStart: 30, + oldCount: 1, + newStart: 30, + newCount: 3 + }] + }, + reason: "충돌 상태 갱신" + }) + assert.doesNotMatch(JSON.stringify(artifact), /consumerSourceMarker/) +}) diff --git a/tests/workflows/mergeRiskWatch.test.ts b/tests/workflows/mergeRiskWatch.test.ts index 2c99cb0..9e3cfcd 100644 --- a/tests/workflows/mergeRiskWatch.test.ts +++ b/tests/workflows/mergeRiskWatch.test.ts @@ -74,6 +74,7 @@ test("writes merge risk debug artifacts", async () => { const originalOpenAiApiKey = process.env.OPENAI_API_KEY const originalDiscordWebhookUrl = process.env.DISCORD_WEBHOOK_URL let openAiRequestCount = 0 + let openAiUserPrompt: string | undefined process.env.OPENAI_API_KEY = "openai-secret" process.env.DISCORD_WEBHOOK_URL = "https://discord.test/webhook-secret" @@ -89,6 +90,15 @@ test("writes merge risk debug artifacts", async () => { openAiRequestCount += 1 assert.equal(request.url, "https://api.openai.com/v1/responses") assert.equal(request.headers.get("Authorization"), "Bearer openai-secret") + const requestBody = await request.json() as { + input?: Array<{ + role?: string + content?: string + }> + } + openAiUserPrompt = requestBody.input + ?.find(item => item.role === "user") + ?.content return jsonResponse({ output_text: JSON.stringify({ @@ -143,6 +153,18 @@ test("writes merge risk debug artifacts", async () => { branchName?: string }> }>(fixture.debugArtifactDir, "ai-result.json") + const aiPromptArtifact = await readJson<{ + prompt?: { + userPrompt?: string + } + }>(fixture.debugArtifactDir, "ai-prompt.json") + const aiResponseArtifact = await readJson<{ + response?: { + predictions?: Array<{ + prediction?: string + }> + } + }>(fixture.debugArtifactDir, "ai-response.json") const deterministicArtifact = await readJson<{ risks?: Array<{ branchName?: string @@ -174,6 +196,14 @@ test("writes merge risk debug artifacts", async () => { ))).join("\n") assert.equal(openAiRequestCount, 1) + assert.ok(openAiUserPrompt) + assert.ok(aiPromptArtifact.prompt) + assert.ok(aiPromptArtifact.prompt.userPrompt) + assert.equal(aiPromptArtifact.prompt.userPrompt, openAiUserPrompt) + assert.equal( + aiResponseArtifact.response?.predictions?.[0]?.prediction, + "critical file update needs review" + ) assert.equal(runArtifact.repository, "opficdev/Watcher") assert.equal(runArtifact.baseBranch, "main") assert.equal(