feat(vscode-lm): support image input conversion and shim data parts

packages/vscode-shim/src/__tests__/Additional.test.ts

@@ -7,6 +7,7 @@ import {
 	CodeActionKind,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,
@@ -261,6 +262,63 @@ describe("LanguageModelTextPart", () => {
 	})
 })
 
+describe("LanguageModelDataPart", () => {
+	const decode = (data: Uint8Array) => new TextDecoder().decode(data)
+
+	it("should create data part via constructor", () => {
+		const data = new Uint8Array([1, 2, 3])
+		const part = new LanguageModelDataPart(data, "image/png")
+
+		expect(part.data).toBe(data)
+		expect(part.mimeType).toBe("image/png")
+	})
+
+	describe("image()", () => {
+		it("should create image part preserving bytes and mime type", () => {
+			const data = new Uint8Array([137, 80, 78, 71])
+			const part = LanguageModelDataPart.image(data, "image/png")
+
+			expect(part).toBeInstanceOf(LanguageModelDataPart)
+			expect(part.data).toEqual(data)
+			expect(part.mimeType).toBe("image/png")
+		})
+
+		it("should round-trip base64-decoded image data", () => {
+			const base64 = Buffer.from("fake-image-bytes").toString("base64")
+			const part = LanguageModelDataPart.image(new Uint8Array(Buffer.from(base64, "base64")), "image/jpeg")
+
+			expect(decode(part.data)).toBe("fake-image-bytes")
+			expect(part.mimeType).toBe("image/jpeg")
+		})
+	})
+})
+
+describe("LanguageModelDataPart text and JSON", () => {
+	it.each([undefined, "text/html"])("encodes text with MIME type %s", (mimeType) => {
+		const part = LanguageModelDataPart.text("Hello 🌍", mimeType)
+		expect(part).toBeInstanceOf(LanguageModelDataPart)
+		expect(new TextDecoder().decode(part.data)).toBe("Hello 🌍")
+		expect(part.mimeType).toBe(mimeType ?? "text/plain")
+	})
+
+	it.each([undefined, "application/example+json"])("encodes JSON with MIME type %s", (mimeType) => {
+		const value = { message: "Hello 🌍", count: 2 }
+		const part = LanguageModelDataPart.json(value, mimeType)
+		expect(part).toBeInstanceOf(LanguageModelDataPart)
+		expect(JSON.parse(new TextDecoder().decode(part.data))).toEqual(value)
+		expect(part.mimeType).toBe(mimeType ?? "application/json")
+	})
+
+	it.each([undefined, () => 1, Symbol("x")])("rejects non-serializable JSON value %s", (value) => {
+		expect(() => LanguageModelDataPart.json(value)).toThrow(TypeError)
+	})
+
+	it.each([null, 0, false, ""])("accepts serializable falsy JSON value %s", (value) => {
+		const part = LanguageModelDataPart.json(value)
+		expect(JSON.parse(new TextDecoder().decode(part.data))).toBe(value)
+	})
+})
+
 describe("LanguageModelToolCallPart", () => {
 	it("should create tool call part", () => {
 		const part = new LanguageModelToolCallPart("call-123", "searchFiles", { query: "test" })

packages/vscode-shim/src/api/create-vscode-api-mock.ts

@@ -21,6 +21,7 @@ import {
 	CodeActionKind,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,
@@ -161,6 +162,7 @@ export function createVSCodeAPIMock(
 		CancellationTokenSource,
 		CodeLens,
 		LanguageModelTextPart,
+		LanguageModelDataPart,
 		LanguageModelToolCallPart,
 		LanguageModelToolResultPart,
 		ExtensionContext: ExtensionContextImpl,

packages/vscode-shim/src/classes/Additional.ts

@@ -112,6 +112,29 @@
 	constructor(public value: string) {}
 }
 
+export class LanguageModelDataPart {
+	constructor(
+		public data: Uint8Array,
+		public mimeType: string,
+	) {}
+
+	static image(data: Uint8Array, mimeType: string): LanguageModelDataPart {
+		return new LanguageModelDataPart(data, mimeType)
+	}
+
+	static text(value: string, mimeType: string = "text/plain"): LanguageModelDataPart {
+		return new LanguageModelDataPart(new TextEncoder().encode(value), mimeType)
+	}
+
+	static json(value: unknown, mimeType: string = "application/json"): LanguageModelDataPart {
+		const serialized = JSON.stringify(value)
+		if (serialized === undefined) {
+			throw new TypeError("LanguageModelDataPart.json: value is not JSON-serializable")
+		}
+		return new LanguageModelDataPart(new TextEncoder().encode(serialized), mimeType)
+	}
+}
+
 export class LanguageModelToolCallPart {
 	constructor(
 		public callId: string,

packages/vscode-shim/src/index.ts

@@ -30,6 +30,7 @@ export {
 	CancellationTokenSource,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,

packages/vscode-shim/src/vscode.ts

@@ -23,6 +23,7 @@ export {
 	CodeActionKind,
 	CodeLens,
 	LanguageModelTextPart,
+	LanguageModelDataPart,
 	LanguageModelToolCallPart,
 	LanguageModelToolResultPart,
 	FileSystemError,

src/api/transform/__tests__/vscode-lm-format.spec.ts

@@ -174,23 +174,48 @@ describe("convertToVsCodeLmMessages", () => {
 		expect(part.content[0]).toEqual({ data: Buffer.from("image"), mimeType: "image/jpeg" })
 	})
 
-	it("does not silently replace unsupported image URLs when image input is enabled", () => {
-		expect(() =>
-			convertToVsCodeLmMessages(
-				[
-					{
-						role: "user",
-						content: [
-							{
-								type: "image",
-								source: { type: "url", url: "https://example.com/image.png" },
-							},
-						],
-					},
-				],
-				true,
-			),
-		).toThrow("requires base64 image data")
+	describe.each(["message", "tool_result"] as const)("image conversion in %s", (path) => {
+		function convertImage(source: Anthropic.ImageBlockParam["source"]) {
+			const image: Anthropic.ImageBlockParam = { type: "image", source }
+			const content: Anthropic.Messages.ContentBlockParam[] =
+				path === "message" ? [image] : [{ type: "tool_result", tool_use_id: "image-tool", content: [image] }]
+			const result = convertToVsCodeLmMessages([{ role: "user", content }], true)
+			const part = result[0].content[0]
+			return part instanceof vscode.LanguageModelToolResultPart ? part.content[0] : part
+		}
+
+		it.each(["image/png", "image/jpeg", "image/gif", "image/webp"] as const)(
+			"sends decoded bytes for %s",
+			(mediaType) => {
+				const bytes = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0xff, 0x00])
+				expect(convertImage({ type: "base64", media_type: mediaType, data: bytes.toString("base64") })).toEqual(
+					{ data: bytes, mimeType: mediaType },
+				)
+			},
+		)
+
+		it("keeps a placeholder for URL images", () => {
+			expect(convertImage({ type: "url", url: "https://example.com/image.png" })).toEqual(
+				new vscode.LanguageModelTextPart("[Image (url): not supported by VSCode LM API]"),
+			)
+		})
+
+		it.each(["", "!!!"])("keeps a placeholder when base64 %j decodes to no bytes", (data) => {
+			expect(convertImage({ type: "base64", media_type: "image/png", data })).toEqual(
+				new vscode.LanguageModelTextPart("[Image (base64): image/png not supported by VSCode LM API]"),
+			)
+		})
+
+		it("keeps a placeholder for unsupported media types", () => {
+			expect(
+				convertImage({
+					type: "base64",
+					// Exercise runtime input outside the SDK's supported media-type union.
+					media_type: "image/bmp" as "image/png",
+					data: "aW1hZ2U=",
+				}),
+			).toEqual(new vscode.LanguageModelTextPart("[Image (base64): image/bmp not supported by VSCode LM API]"))
+		})
 	})
 
 	it("should convert simple string messages", () => {

src/api/transform/vscode-lm-format.ts

@@ -93,16 +93,19 @@
 	return decoded
 }
 
+const SUPPORTED_IMAGE_MEDIA_TYPES = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"])
+
 function convertImageToVsCodeLmPart(image: Anthropic.Messages.ImageBlockParam, supportsImages: boolean) {
-	if (supportsImages) {
-		if (image.source.type !== "base64") {
-			throw new Error("GitHub Copilot image input requires base64 image data, not an image URL.")
-		}
-		const part = createImagePart(Buffer.from(image.source.data, "base64"), image.source.media_type)
-		if (!part) {
-			throw new Error("Image input requires a newer version of VS Code that can send image data.")
+	if (supportsImages && image.source.type === "base64" && SUPPORTED_IMAGE_MEDIA_TYPES.has(image.source.media_type)) {
+		// Pass decoded bytes, not the base64 string, to avoid encoding the image twice.
+		const bytes = Buffer.from(image.source.data, "base64")
+		if (bytes.length > 0) {
+			const part = createImagePart(bytes, image.source.media_type)
+			if (!part) {
+				throw new Error("Image input requires a newer version of VS Code that can send image data.")
+			}
+			return part
 		}
-		return part
 	}
 	return new vscode.LanguageModelTextPart(
 		image.source.type === "base64"