From caa4bd3a9462d3f6192e36f6cd3d3ba54ac4e5dd Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 25 Oct 2026 19:21:24 -0700 Subject: [PATCH 1/2] feat(vscode-lm): support image input conversion and shim data parts --- .../src/__tests__/Additional.test.ts | 49 +++++++++++++++ .../src/api/create-vscode-api-mock.ts | 2 + .../vscode-shim/src/classes/Additional.ts | 19 ++++++ packages/vscode-shim/src/index.ts | 1 + packages/vscode-shim/src/vscode.ts | 1 + .../__tests__/vscode-lm-format.spec.ts | 59 +++++++++++++------ src/api/transform/vscode-lm-format.ts | 19 +++--- 7 files changed, 125 insertions(+), 25 deletions(-) diff --git a/packages/vscode-shim/src/__tests__/Additional.test.ts b/packages/vscode-shim/src/__tests__/Additional.test.ts index 7f1abbee14..da80cd4aea 100644 --- a/packages/vscode-shim/src/__tests__/Additional.test.ts +++ b/packages/vscode-shim/src/__tests__/Additional.test.ts @@ -7,6 +7,7 @@ import { CodeActionKind, CodeLens, LanguageModelTextPart, + LanguageModelDataPart, LanguageModelToolCallPart, LanguageModelToolResultPart, FileSystemError, @@ -261,6 +262,54 @@ describe("LanguageModelTextPart", () => { }) }) +describe("LanguageModelDataPart", () => { + const decode = (data: Uint8Array) => new TextDecoder().decode(data) + + it("should create data part via constructor", () => { + const data = new Uint8Array([1, 2, 3]) + const part = new LanguageModelDataPart(data, "image/png") + + expect(part.data).toBe(data) + expect(part.mimeType).toBe("image/png") + }) + + describe("image()", () => { + it("should create image part preserving bytes and mime type", () => { + const data = new Uint8Array([137, 80, 78, 71]) + const part = LanguageModelDataPart.image(data, "image/png") + + expect(part).toBeInstanceOf(LanguageModelDataPart) + expect(part.data).toEqual(data) + expect(part.mimeType).toBe("image/png") + }) + + it("should round-trip base64-decoded image data", () => { + const base64 = Buffer.from("fake-image-bytes").toString("base64") + const part = LanguageModelDataPart.image(new Uint8Array(Buffer.from(base64, "base64")), "image/jpeg") + + expect(decode(part.data)).toBe("fake-image-bytes") + expect(part.mimeType).toBe("image/jpeg") + }) + }) +}) + +describe("LanguageModelDataPart text and JSON", () => { + it.each([undefined, "text/html"])("encodes text with MIME type %s", (mimeType) => { + const part = LanguageModelDataPart.text("Hello 🌍", mimeType) + expect(part).toBeInstanceOf(LanguageModelDataPart) + expect(new TextDecoder().decode(part.data)).toBe("Hello 🌍") + expect(part.mimeType).toBe(mimeType ?? "text/plain") + }) + + it.each([undefined, "application/example+json"])("encodes JSON with MIME type %s", (mimeType) => { + const value = { message: "Hello 🌍", count: 2 } + const part = LanguageModelDataPart.json(value, mimeType) + expect(part).toBeInstanceOf(LanguageModelDataPart) + expect(JSON.parse(new TextDecoder().decode(part.data))).toEqual(value) + expect(part.mimeType).toBe(mimeType ?? "application/json") + }) +}) + describe("LanguageModelToolCallPart", () => { it("should create tool call part", () => { const part = new LanguageModelToolCallPart("call-123", "searchFiles", { query: "test" }) diff --git a/packages/vscode-shim/src/api/create-vscode-api-mock.ts b/packages/vscode-shim/src/api/create-vscode-api-mock.ts index fd4a94a8a6..f686a5147f 100644 --- a/packages/vscode-shim/src/api/create-vscode-api-mock.ts +++ b/packages/vscode-shim/src/api/create-vscode-api-mock.ts @@ -21,6 +21,7 @@ import { CodeActionKind, CodeLens, LanguageModelTextPart, + LanguageModelDataPart, LanguageModelToolCallPart, LanguageModelToolResultPart, FileSystemError, @@ -161,6 +162,7 @@ export function createVSCodeAPIMock( CancellationTokenSource, CodeLens, LanguageModelTextPart, + LanguageModelDataPart, LanguageModelToolCallPart, LanguageModelToolResultPart, ExtensionContext: ExtensionContextImpl, diff --git a/packages/vscode-shim/src/classes/Additional.ts b/packages/vscode-shim/src/classes/Additional.ts index d300eb1e8c..f12e89404f 100644 --- a/packages/vscode-shim/src/classes/Additional.ts +++ b/packages/vscode-shim/src/classes/Additional.ts @@ -112,6 +112,25 @@ export class LanguageModelTextPart { constructor(public value: string) {} } +export class LanguageModelDataPart { + constructor( + public data: Uint8Array, + public mimeType: string, + ) {} + + static image(data: Uint8Array, mimeType: string): LanguageModelDataPart { + return new LanguageModelDataPart(data, mimeType) + } + + static text(value: string, mimeType: string = "text/plain"): LanguageModelDataPart { + return new LanguageModelDataPart(new TextEncoder().encode(value), mimeType) + } + + static json(value: unknown, mimeType: string = "application/json"): LanguageModelDataPart { + return new LanguageModelDataPart(new TextEncoder().encode(JSON.stringify(value)), mimeType) + } +} + export class LanguageModelToolCallPart { constructor( public callId: string, diff --git a/packages/vscode-shim/src/index.ts b/packages/vscode-shim/src/index.ts index 8f40746de7..4c72dba056 100644 --- a/packages/vscode-shim/src/index.ts +++ b/packages/vscode-shim/src/index.ts @@ -30,6 +30,7 @@ export { CancellationTokenSource, CodeLens, LanguageModelTextPart, + LanguageModelDataPart, LanguageModelToolCallPart, LanguageModelToolResultPart, FileSystemError, diff --git a/packages/vscode-shim/src/vscode.ts b/packages/vscode-shim/src/vscode.ts index a25cd1e8d9..a25677a6ea 100644 --- a/packages/vscode-shim/src/vscode.ts +++ b/packages/vscode-shim/src/vscode.ts @@ -23,6 +23,7 @@ export { CodeActionKind, CodeLens, LanguageModelTextPart, + LanguageModelDataPart, LanguageModelToolCallPart, LanguageModelToolResultPart, FileSystemError, diff --git a/src/api/transform/__tests__/vscode-lm-format.spec.ts b/src/api/transform/__tests__/vscode-lm-format.spec.ts index e66e3f869f..30654a0112 100644 --- a/src/api/transform/__tests__/vscode-lm-format.spec.ts +++ b/src/api/transform/__tests__/vscode-lm-format.spec.ts @@ -174,23 +174,48 @@ describe("convertToVsCodeLmMessages", () => { expect(part.content[0]).toEqual({ data: Buffer.from("image"), mimeType: "image/jpeg" }) }) - it("does not silently replace unsupported image URLs when image input is enabled", () => { - expect(() => - convertToVsCodeLmMessages( - [ - { - role: "user", - content: [ - { - type: "image", - source: { type: "url", url: "https://example.com/image.png" }, - }, - ], - }, - ], - true, - ), - ).toThrow("requires base64 image data") + describe.each(["message", "tool_result"] as const)("image conversion in %s", (path) => { + function convertImage(source: Anthropic.ImageBlockParam["source"]) { + const image: Anthropic.ImageBlockParam = { type: "image", source } + const content: Anthropic.Messages.ContentBlockParam[] = + path === "message" ? [image] : [{ type: "tool_result", tool_use_id: "image-tool", content: [image] }] + const result = convertToVsCodeLmMessages([{ role: "user", content }], true) + const part = result[0].content[0] + return part instanceof vscode.LanguageModelToolResultPart ? part.content[0] : part + } + + it.each(["image/png", "image/jpeg", "image/gif", "image/webp"] as const)( + "sends decoded bytes for %s", + (mediaType) => { + const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0xff, 0x00]) + expect(convertImage({ type: "base64", media_type: mediaType, data: bytes.toString("base64") })).toEqual( + { data: bytes, mimeType: mediaType }, + ) + }, + ) + + it("keeps a placeholder for URL images", () => { + expect(convertImage({ type: "url", url: "https://example.com/image.png" })).toEqual( + new vscode.LanguageModelTextPart("[Image (url): not supported by VSCode LM API]"), + ) + }) + + it.each(["", "!!!"])("keeps a placeholder when base64 %j decodes to no bytes", (data) => { + expect(convertImage({ type: "base64", media_type: "image/png", data })).toEqual( + new vscode.LanguageModelTextPart("[Image (base64): image/png not supported by VSCode LM API]"), + ) + }) + + it("keeps a placeholder for unsupported media types", () => { + expect( + convertImage({ + type: "base64", + // Exercise runtime input outside the SDK's supported media-type union. + media_type: "image/bmp" as "image/png", + data: "aW1hZ2U=", + }), + ).toEqual(new vscode.LanguageModelTextPart("[Image (base64): image/bmp not supported by VSCode LM API]")) + }) }) it("should convert simple string messages", () => { diff --git a/src/api/transform/vscode-lm-format.ts b/src/api/transform/vscode-lm-format.ts index 8400145132..cb0f8e1aa9 100644 --- a/src/api/transform/vscode-lm-format.ts +++ b/src/api/transform/vscode-lm-format.ts @@ -93,16 +93,19 @@ export function decodeToolNameSurrogates(name: string): string { return decoded } +const SUPPORTED_IMAGE_MEDIA_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]) + function convertImageToVsCodeLmPart(image: Anthropic.Messages.ImageBlockParam, supportsImages: boolean) { - if (supportsImages) { - if (image.source.type !== "base64") { - throw new Error("GitHub Copilot image input requires base64 image data, not an image URL.") - } - const part = createImagePart(Buffer.from(image.source.data, "base64"), image.source.media_type) - if (!part) { - throw new Error("Image input requires a newer version of VS Code that can send image data.") + if (supportsImages && image.source.type === "base64" && SUPPORTED_IMAGE_MEDIA_TYPES.has(image.source.media_type)) { + // Pass decoded bytes, not the base64 string, to avoid encoding the image twice. + const bytes = Buffer.from(image.source.data, "base64") + if (bytes.length > 0) { + const part = createImagePart(bytes, image.source.media_type) + if (!part) { + throw new Error("Image input requires a newer version of VS Code that can send image data.") + } + return part } - return part } return new vscode.LanguageModelTextPart( image.source.type === "base64" From f82d89ebb1a8ba365ef14b851aa7e79920b4d36c Mon Sep 17 00:00:00 2001 From: Bertan Ari Date: Sun, 25 Oct 2026 21:37:04 -0700 Subject: [PATCH 2/2] fix(vscode-shim): reject non-serializable values in LanguageModelDataPart.json --- packages/vscode-shim/src/__tests__/Additional.test.ts | 9 +++++++++ packages/vscode-shim/src/classes/Additional.ts | 6 +++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/vscode-shim/src/__tests__/Additional.test.ts b/packages/vscode-shim/src/__tests__/Additional.test.ts index da80cd4aea..0652779016 100644 --- a/packages/vscode-shim/src/__tests__/Additional.test.ts +++ b/packages/vscode-shim/src/__tests__/Additional.test.ts @@ -308,6 +308,15 @@ describe("LanguageModelDataPart text and JSON", () => { expect(JSON.parse(new TextDecoder().decode(part.data))).toEqual(value) expect(part.mimeType).toBe(mimeType ?? "application/json") }) + + it.each([undefined, () => 1, Symbol("x")])("rejects non-serializable JSON value %s", (value) => { + expect(() => LanguageModelDataPart.json(value)).toThrow(TypeError) + }) + + it.each([null, 0, false, ""])("accepts serializable falsy JSON value %s", (value) => { + const part = LanguageModelDataPart.json(value) + expect(JSON.parse(new TextDecoder().decode(part.data))).toBe(value) + }) }) describe("LanguageModelToolCallPart", () => { diff --git a/packages/vscode-shim/src/classes/Additional.ts b/packages/vscode-shim/src/classes/Additional.ts index f12e89404f..0b23bec191 100644 --- a/packages/vscode-shim/src/classes/Additional.ts +++ b/packages/vscode-shim/src/classes/Additional.ts @@ -127,7 +127,11 @@ export class LanguageModelDataPart { } static json(value: unknown, mimeType: string = "application/json"): LanguageModelDataPart { - return new LanguageModelDataPart(new TextEncoder().encode(JSON.stringify(value)), mimeType) + const serialized = JSON.stringify(value) + if (serialized === undefined) { + throw new TypeError("LanguageModelDataPart.json: value is not JSON-serializable") + } + return new LanguageModelDataPart(new TextEncoder().encode(serialized), mimeType) } }