From a2eaa5ae1747dde8fdecf505def8c37fc3a6d242 Mon Sep 17 00:00:00 2001 From: Antisophy <293439221+Antisophy@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:06:20 -0700 Subject: [PATCH] feat(web): accept HEIC photos, transcoding them to JPEG on the server An iPhone camera photo is HEIC. When one is dragged or pasted into the composer, Gecko has no MIME entry for .heic, so the File arrives with an empty type; the composer then dropped it without a word, since nothing outside image/* was ever attached. A photo that vanishes with no trace is indistinguishable from a broken composer, which is how it presented. A browser that does type the file image/heic attached it as-is, producing a send the agent APIs reject. Client: sniff the leading bytes when the browser could not type a file (JPEG, PNG, GIF, WebP and the HEIC/HEIF ftyp brands), accept HEIC, show a labelled chip in place of the preview the browser cannot decode, and name any file that is refused. Files the browser typed as a supported format stay on the synchronous path. Server: a new cydo.web.images normalizes image blocks before the create_task and message handlers see them, transcoding HEIC to JPEG with ImageMagick, resolved through PATH at call time so a missing install degrades to a clear client-visible error. The conversion is asynchronous, since the event loop is shared by every client; when nothing needs converting the handler runs synchronously so ordering is unchanged, and a conversion failure is reported to the client rather than forwarded to the agent. Each conversion is issued from a helper call whose parameter is a fresh variable per call, so several photos in one message each land in their own JPEG block. The unit test drives real HEIC bytes through the pipeline, and imagemagick joins the unittests check environment so it runs in the sandbox rather than skipping itself. --- flake.nix | 5 +- source/cydo/server/app.d | 34 ++- source/cydo/web/images.d | 233 ++++++++++++++++++++ web/src/components/InputBox.attach.test.tsx | 139 ++++++++++++ web/src/components/InputBox.tsx | 103 ++++++++- web/src/styles.css | 23 ++ 6 files changed, 531 insertions(+), 6 deletions(-) create mode 100644 source/cydo/web/images.d create mode 100644 web/src/components/InputBox.attach.test.tsx diff --git a/flake.nix b/flake.nix index b1ca8c41..c33d31aa 100644 --- a/flake.nix +++ b/flake.nix @@ -963,7 +963,10 @@ EOF dubLock = ./dub-lock.json; - nativeBuildInputs = [ pkgs.git pkgs.pkg-config ]; + # imagemagick (with HEIC support in nixpkgs) lets the HEIC + # transcoding unit test run for real; it skips itself where + # `magick` cannot produce HEIC + nativeBuildInputs = [ pkgs.git pkgs.pkg-config pkgs.imagemagick ]; buildInputs = [ pkgs.sqlite pkgs.openssl pkgs.zlib ]; CI = "1"; diff --git a/source/cydo/server/app.d b/source/cydo/server/app.d index c127f5ba..7deb76cb 100644 --- a/source/cydo/server/app.d +++ b/source/cydo/server/app.d @@ -38,6 +38,7 @@ import cydo.workflow.workspace.archive_manager : ArchiveManager, ArchiveManagerH import cydo.workflow.workspace.task_path_resolver : TaskPathResolver, TaskPathResolverHost; import cydo.workflow.workspace.worktree_allocator : WorktreeAllocator, WorktreeAllocatorHost; import cydo.web.client_hub : ClientHub; +import cydo.web.images : ImageNormalization, normalizeImageBlocks; import cydo.runtime.config.watcher : ConfigWatcher, ConfigWatcherHost; import cydo.workflow.discovery.service : DiscoveryService, DiscoveryServiceHost, DiscoveryTaskSnapshot, ImportableReconciliationCommit, ImportableScanRecord, @@ -1263,9 +1264,9 @@ class App switch (json.type) { - case "create_task": handleCreateTaskMsg(ws, json); break; + case "create_task": withNormalizedImages(ws, json, (WsMessage m) { handleCreateTaskMsg(ws, m); }); break; case "request_history": handleRequestHistory(ws, json); break; - case "message": handleUserMessage(json); break; + case "message": withNormalizedImages(ws, json, (WsMessage m) { handleUserMessage(m); }); break; case "resume": handleResumeMsg(json); break; case "interrupt": handleInterruptMsg(json); break; case "sigint": handleSigintMsg(json); break; @@ -1458,6 +1459,35 @@ class App } } + /// Run a content-carrying handler with its image blocks made acceptable to + /// the agent APIs first. Synchronous when nothing needs converting, so the + /// common path keeps its ordering; a message whose photos need transcoding + /// is dispatched once the conversions finish, and is refused with a visible + /// error rather than forwarded if one of them cannot be converted. + private void withNormalizedImages(WebSocketAdapter ws, WsMessage json, + void delegate(WsMessage) handler) + { + import ae.utils.json : jsonParse, toJson, JSONFragment; + + if (json.content.json is null) + { + handler(json); + return; + } + auto blocks = jsonParse!(ContentBlock[])(json.content.json); + normalizeImageBlocks(blocks, (ImageNormalization normalized) { + if (normalized.error.length > 0) + { + import std.logger : warningf; + warningf("image attachment rejected for tid=%d: %s", json.tid, normalized.error); + ws.send(Data(toJson(ErrorMessage("error", normalized.error, json.tid)).representation)); + return; + } + json.content = JSONFragment(toJson(normalized.blocks)); + handler(json); + }); + } + private void handleRequestHistory(WebSocketAdapter ws, WsMessage json) { historyPipeline.handleRequestHistory(ws, json.tid); diff --git a/source/cydo/web/images.d b/source/cydo/web/images.d new file mode 100644 index 00000000..ee0bdb8b --- /dev/null +++ b/source/cydo/web/images.d @@ -0,0 +1,233 @@ +/// Image blocks arriving from a browser, normalized into what the agent APIs +/// accept before any handler sees them. +/// +/// The case that made this necessary: an iPhone camera photo is HEIC, and a +/// mobile browser hands it over as the original file. No agent API takes HEIC, +/// so the block is transcoded to JPEG here with ImageMagick. The conversion is +/// asynchronous: the event loop is single-threaded and shared by every client, +/// so blocking it for the second a transcode takes would stall everyone's +/// streaming. +module cydo.web.images; + +import std.base64 : Base64; +import std.process : Config, Pid, pipe, spawnProcess; +import std.stdio : File; + +import ae.net.asockets : DisconnectType, FileConnection; +import ae.sys.data : Data; +import ae.sys.process : asyncWait; +import ae.utils.array : asBytes; + +import cydo.protocol : ContentBlock; + +/// media types the agent APIs accept as-is +bool isNativeImageType(string mediaType) +{ + switch (mediaType) + { + case "image/jpeg", "image/png", "image/gif", "image/webp": return true; + default: return false; + } +} + +/// media types this module can turn into a native one +bool isConvertibleImageType(string mediaType) +{ + return mediaType == "image/heic" || mediaType == "image/heif"; +} + +/// Outcome of normalizing one message's blocks. +struct ImageNormalization +{ + ContentBlock[] blocks; + string error; /// non-empty when a block could not be made acceptable +} + +/// Rewrite any convertible image block in `blocks` and hand the result to +/// `done`. Calls `done` synchronously when nothing needs converting, so the +/// ordinary path keeps its ordering guarantees; otherwise `done` runs once +/// every conversion has finished. +void normalizeImageBlocks(ContentBlock[] blocks, void delegate(ImageNormalization) done) +{ + size_t pending = 0; + foreach (ref b; blocks) + if (b.type == "image" && isConvertibleImageType(b.media_type)) + pending++; + if (pending == 0) + { + done(ImageNormalization(blocks, null)); + return; + } + + auto result = ImageNormalization(blocks.dup, null); + size_t remaining = pending; + + // one frame per conversion: a delegate made in a loop body shares the + // function's single closure frame, so every one of them would otherwise + // see the final loop index and write into the same block. a parameter of + // a helper call is a fresh variable each time + void convertBlock(size_t index) + { + convertToJpeg(Base64.decode(result.blocks[index].data), (ubyte[] jpeg, string error) { + if (error.length > 0) + { + if (result.error.length == 0) + result.error = error; + } + else + { + result.blocks[index].data = Base64.encode(jpeg).idup; + result.blocks[index].media_type = "image/jpeg"; + } + if (--remaining == 0) + done(result); + }); + } + + foreach (i, ref b; result.blocks) + if (b.type == "image" && isConvertibleImageType(b.media_type)) + convertBlock(i); +} + +/// Transcode one image to JPEG with ImageMagick, asynchronously. +/// +/// `magick` is resolved through PATH at call time, so a missing install is an +/// error for the one attachment rather than a startup failure for the server. +void convertToJpeg(ubyte[] input, void delegate(ubyte[] jpeg, string error) done) +{ + import std.process : environment; + import std.file : exists; + import std.path : buildPath; + import std.algorithm : splitter; + + string magick; + foreach (dir; environment.get("PATH", "").splitter(':')) + { + auto candidate = buildPath(dir, "magick"); + if (candidate.exists) + { + magick = candidate; + break; + } + } + if (magick.length == 0) + { + done(null, "cannot convert this image: ImageMagick (magick) is not installed on the server"); + return; + } + + auto stdinPipe = pipe(); + auto stdoutPipe = pipe(); + auto stderrPipe = pipe(); + // quality 90 keeps a phone photo well under the agent APIs' size limits + // while staying visually lossless for their purposes + auto pid = spawnProcess( + [magick, "-", "-auto-orient", "-quality", "90", "jpeg:-"], + stdinPipe.readEnd, stdoutPipe.writeEnd, stderrPipe.writeEnd, + null, Config.none); + stdinPipe.readEnd.close(); + stdoutPipe.writeEnd.close(); + stderrPipe.writeEnd.close(); + + import core.sys.posix.unistd : dup; + auto stdinConn = new FileConnection(dup(stdinPipe.writeEnd.fileno)); + auto stdoutConn = new FileConnection(dup(stdoutPipe.readEnd.fileno)); + auto stderrConn = new FileConnection(dup(stderrPipe.readEnd.fileno)); + stdinPipe.writeEnd.close(); + stdoutPipe.readEnd.close(); + stderrPipe.readEnd.close(); + + ubyte[] output; + ubyte[] errors; + bool stdoutDone, stderrDone, exited; + int status; + bool finished; + + void finish() + { + if (finished || !stdoutDone || !stderrDone || !exited) + return; + finished = true; + if (status != 0 || output.length == 0) + { + import std.string : strip; + auto detail = (cast(string) errors).strip; + done(null, "cannot convert this image to JPEG" + ~ (detail.length ? ": " ~ detail : "")); + } + else + done(output, null); + } + + stdoutConn.handleReadData = (Data data) { output ~= cast(ubyte[]) data.toGC(); }; + stdoutConn.handleDisconnect = (string, DisconnectType) { stdoutDone = true; finish(); }; + stderrConn.handleReadData = (Data data) { errors ~= cast(ubyte[]) data.toGC(); }; + stderrConn.handleDisconnect = (string, DisconnectType) { stderrDone = true; finish(); }; + asyncWait(pid, (int code) { status = code; exited = true; finish(); }); + + // a requested disconnect on a connection with queued writes closes the fd + // only after everything has been flushed, which is what hands magick its EOF + stdinConn.send(Data(input)); + stdinConn.disconnect("input written"); +} + +unittest +{ + import std.process : environment, execute; + import ae.net.asockets : socketManager; + import ae.utils.json : JSONFragment; + + // needs a real ImageMagick with HEIC support; the nix sandbox has none, so + // the test proves the pipeline where it can and is silent where it cannot + auto probe = execute(["sh", "-c", "command -v magick >/dev/null 2>&1 && magick -list format | grep -q HEIC"]); + if (probe.status != 0) + return; + + auto heic = execute(["magick", "-size", "8x6", "xc:tomato", "heic:-"]); + assert(heic.status == 0 && heic.output.length > 12, "could not produce a HEIC fixture"); + + auto blocks = [ + ContentBlock("text", "caption"), + ContentBlock("image", null, null, null, JSONFragment.init, + Base64.encode(cast(ubyte[]) heic.output).idup, "image/heic"), + ]; + ImageNormalization got; + bool called; + normalizeImageBlocks(blocks, (ImageNormalization r) { got = r; called = true; }); + assert(!called, "a convertible block must not complete synchronously"); + while (!called) + socketManager.loop(); + assert(got.error.length == 0, got.error); + assert(got.blocks[0].type == "text" && got.blocks[0].text == "caption", "other blocks pass through untouched"); + assert(got.blocks[1].media_type == "image/jpeg"); + auto jpeg = Base64.decode(got.blocks[1].data); + assert(jpeg.length > 2 && jpeg[0] == 0xFF && jpeg[1] == 0xD8, "output must be a JPEG"); + + // two photos in one message: each must land in its own block. a single + // shared closure frame made both conversions write the second slot, so the + // first went to the agent still as HEIC + auto second = execute(["magick", "-size", "12x9", "xc:navy", "heic:-"]); + assert(second.status == 0); + auto pair = [ + ContentBlock("image", null, null, null, JSONFragment.init, + Base64.encode(cast(ubyte[]) heic.output).idup, "image/heic"), + ContentBlock("text", "between"), + ContentBlock("image", null, null, null, JSONFragment.init, + Base64.encode(cast(ubyte[]) second.output).idup, "image/heic"), + ]; + ImageNormalization both; + bool pairDone; + normalizeImageBlocks(pair, (ImageNormalization r) { both = r; pairDone = true; }); + while (!pairDone) + socketManager.loop(); + assert(both.error.length == 0, both.error); + assert(both.blocks[0].media_type == "image/jpeg" && both.blocks[2].media_type == "image/jpeg", + "every convertible block is converted, not just the last"); + assert(both.blocks[0].data != both.blocks[2].data, "each block keeps its own image"); + assert(both.blocks[1].text == "between"); + + // nothing convertible: the continuation runs synchronously, in order + bool sync; + normalizeImageBlocks([ContentBlock("text", "plain")], (ImageNormalization r) { sync = true; }); + assert(sync, "an unconvertible batch must complete synchronously"); +} diff --git a/web/src/components/InputBox.attach.test.tsx b/web/src/components/InputBox.attach.test.tsx new file mode 100644 index 00000000..b9add992 --- /dev/null +++ b/web/src/components/InputBox.attach.test.tsx @@ -0,0 +1,139 @@ +/** + * @vitest-environment jsdom + * @vitest-environment-options { "pretendToBeVisual": true } + * + * Image intake through the composer: every file handed over must either + * become a visible attachment or a visible refusal. A photo that vanishes + * with no trace is indistinguishable from a broken composer. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { render } from "preact"; +import { act } from "preact/test-utils"; +import { InputBox, drafts } from "./InputBox"; + +vi.hoisted(() => { + vi.stubGlobal("CSS", { supports: () => false }); +}); + +const SESSION = "attach-session"; + +/** FileReader.readAsDataURL and the type sniffer resolve on tasks; let them land. */ +const flushAsync = () => new Promise((resolve) => setTimeout(resolve, 60)); + +async function mountComposer(container: HTMLElement) { + await act(() => { + render( + {}} + onInterrupt={() => {}} + isProcessing={false} + stdinClosed={false} + disabled={false} + sessionId={SESSION} + />, + container, + ); + }); +} + +/** A File with the leading bytes of the named format, typed as the browser + * would type it (an empty type is what Gecko assigns to .heic). */ +function imageFile(name: string, type: string, magic?: number[]) { + const bytes = magic ?? + { + "image/jpeg": [0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46], + "image/png": [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], + }[type] ?? [1, 2, 3, 4]; + return new File([new Uint8Array(bytes)], name, { type }); +} + +/** bytes 4..12 of an iPhone camera HEIC: "ftypheic" */ +const HEIC_MAGIC = [ + 0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, +]; + +/** Deliver files the way a drag-and-drop does. */ +async function dropFiles(container: HTMLElement, files: File[]) { + const box = container.querySelector(".input-box"); + expect(box).not.toBeNull(); + // a FileList stand-in: indexed access, length, item(), and iteration + const list: Record = { + item: (index: number) => files[index] ?? null, + [Symbol.iterator]: files[Symbol.iterator].bind(files), + }; + files.forEach((file, index) => { + list[index] = file; + }); + list.length = files.length; + const event = new Event("drop", { bubbles: true, cancelable: true }); + Object.defineProperty(event, "dataTransfer", { + value: { files: list, types: ["Files"] }, + }); + await act(() => { + box!.dispatchEvent(event); + }); +} + +describe("composer image attachment", () => { + let container: HTMLElement; + + beforeEach(() => { + drafts.clear(); + container = document.createElement("div"); + document.body.appendChild(container); + }); + + it("attaches dropped images as previews", async () => { + await mountComposer(container); + await dropFiles(container, [ + imageFile("one.jpg", "image/jpeg"), + imageFile("two.png", "image/png"), + ]); + await act(async () => { + await flushAsync(); + }); + + expect(container.querySelectorAll(".image-preview img").length).toBe(2); + }); + + it("refuses image types nothing downstream can use, and says so", async () => { + await mountComposer(container); + await dropFiles(container, [imageFile("scan.bmp", "image/bmp")]); + await act(async () => { + await flushAsync(); + }); + + expect(container.querySelectorAll(".image-preview").length).toBe(0); + const error = container.querySelector(".attach-error"); + expect(error).not.toBeNull(); + expect(error!.textContent).toContain("image/bmp"); + }); + + it("recognizes a HEIC photo by its bytes even with no browser-assigned type", async () => { + await mountComposer(container); + // gecko has no mime entry for .heic, so the File arrives with type "" + await dropFiles(container, [imageFile("IMG_0001.HEIC", "", HEIC_MAGIC)]); + await act(async () => { + await flushAsync(); + }); + + // nothing silently dropped: a chip stands in for the undecodable preview + expect(container.querySelector(".attach-error")).toBeNull(); + const label = container.querySelector(".image-preview-label"); + expect(label).not.toBeNull(); + expect(label!.textContent).toContain("HEIC"); + expect(container.querySelectorAll(".image-preview img").length).toBe(0); + }); + + it("names an unrecognized file instead of dropping it silently", async () => { + await mountComposer(container); + await dropFiles(container, [imageFile("mystery.bin", "", [0, 1, 2, 3])]); + await act(async () => { + await flushAsync(); + }); + + const error = container.querySelector(".attach-error"); + expect(error).not.toBeNull(); + expect(error!.textContent).toContain("mystery.bin"); + }); +}); diff --git a/web/src/components/InputBox.tsx b/web/src/components/InputBox.tsx index 636f49e9..83196e5e 100644 --- a/web/src/components/InputBox.tsx +++ b/web/src/components/InputBox.tsx @@ -132,6 +132,45 @@ function publishImages(entry: ControlledImageEntry, images: ImageAttachment[]) { } } +// what the agent APIs accept as-is, plus what the server transcodes for them: +// an iPhone camera photo is HEIC, and a mobile browser hands over the original +const SUPPORTED_IMAGE_TYPES = new Set([ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + "image/heic", + "image/heif", +]); + +// formats the browser itself cannot decode, so the preview shows a label +// instead of a broken ; the server turns them into JPEG before sending +const PREVIEW_UNDECODABLE = new Set(["image/heic", "image/heif"]); + +/** The real format from the file's leading bytes. + * + * `File.type` comes from the browser's extension table, and a dragged or + * pasted file can arrive with no usable extension or one the browser has no + * entry for (Gecko has none for .heic), which leaves `type` empty. The bytes + * do not lie. + */ +async function sniffImageType(file: File): Promise { + const head = new Uint8Array(await file.slice(0, 16).arrayBuffer()); + const ascii = (from: number, to: number) => + String.fromCharCode(...head.subarray(from, to)); + if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) + return "image/jpeg"; + if (head[0] === 0x89 && ascii(1, 4) === "PNG") return "image/png"; + if (ascii(0, 4) === "GIF8") return "image/gif"; + if (ascii(0, 4) === "RIFF" && ascii(8, 12) === "WEBP") return "image/webp"; + if (ascii(4, 8) === "ftyp") { + const brand = ascii(8, 12); + if (["heic", "heix", "hevc", "hevx", "mif1", "msf1"].includes(brand)) + return "image/heic"; + } + return file.type; +} + function useImageAttachments( enabled = true, resetToken?: number, @@ -151,6 +190,7 @@ function useImageAttachments( () => imageEntry?.generation ?? 0, ); const [isDragging, setIsDragging] = useState(false); + const [attachError, setAttachError] = useState(null); const enabledRef = useRef(enabled); const resetTokenRef = useRef(resetToken); const generationRef = useRef(imageEntry?.generation ?? 0); @@ -216,7 +256,30 @@ function useImageAttachments( }; const processFile = (file: File) => { - if (!enabledRef.current || !file.type.startsWith("image/")) return; + if (!enabledRef.current) return; + // a type the browser resolved from a known extension is trustworthy and + // keeps the synchronous path; sniffing is for the files it could not type + if (SUPPORTED_IMAGE_TYPES.has(file.type)) { + attachTyped(file, file.type); + return; + } + void sniffImageType(file).then((mediaType) => { + attachTyped(file, mediaType); + }); + }; + + // nothing here may drop a file silently: a picked photo that vanishes with no + // trace is indistinguishable from a broken attach control + const attachTyped = (file: File, mediaType: string) => { + if (!enabledRef.current) return; + if (!SUPPORTED_IMAGE_TYPES.has(mediaType)) { + const label = mediaType || `"${file.name}" (unrecognized format)`; + setAttachError( + `${label} can't be attached; JPEG, PNG, GIF, WebP and HEIC work`, + ); + return; + } + setAttachError(null); const generation = generationRef.current; const entry = imageEntry; const reader = new FileReader(); @@ -227,7 +290,7 @@ function useImageAttachments( id: crypto.randomUUID(), dataURL, base64, - mediaType: file.type, + mediaType, }; if (entry) { if ( @@ -297,6 +360,10 @@ function useImageAttachments( images: enabled && imagesGeneration === generationRef.current ? images : [], setImages, isDragging: enabled && isDragging, + attachError, + dismissAttachError: () => { + setAttachError(null); + }, onPaste, onDragOver, onDragLeave, @@ -304,6 +371,20 @@ function useImageAttachments( }; } +function AttachError({ + message, + onDismiss, +}: { + message: string; + onDismiss: () => void; +}) { + return ( +
+ {message} +
+ ); +} + function ImagePreviews({ images, onRemove, @@ -318,7 +399,13 @@ function ImagePreviews({
{images.map((image) => (
- Attached + {PREVIEW_UNDECODABLE.has(image.mediaType) ? ( +
+ {image.mediaType.replace("image/", "").toUpperCase()} photo +
+ ) : ( + Attached + )}