diff --git a/flake.nix b/flake.nix index b1ca8c41..c33d31aa 100644 --- a/flake.nix +++ b/flake.nix @@ -963,7 +963,10 @@ EOF dubLock = ./dub-lock.json; - nativeBuildInputs = [ pkgs.git pkgs.pkg-config ]; + # imagemagick (with HEIC support in nixpkgs) lets the HEIC + # transcoding unit test run for real; it skips itself where + # `magick` cannot produce HEIC + nativeBuildInputs = [ pkgs.git pkgs.pkg-config pkgs.imagemagick ]; buildInputs = [ pkgs.sqlite pkgs.openssl pkgs.zlib ]; CI = "1"; diff --git a/source/cydo/server/app.d b/source/cydo/server/app.d index c127f5ba..7deb76cb 100644 --- a/source/cydo/server/app.d +++ b/source/cydo/server/app.d @@ -38,6 +38,7 @@ import cydo.workflow.workspace.archive_manager : ArchiveManager, ArchiveManagerH import cydo.workflow.workspace.task_path_resolver : TaskPathResolver, TaskPathResolverHost; import cydo.workflow.workspace.worktree_allocator : WorktreeAllocator, WorktreeAllocatorHost; import cydo.web.client_hub : ClientHub; +import cydo.web.images : ImageNormalization, normalizeImageBlocks; import cydo.runtime.config.watcher : ConfigWatcher, ConfigWatcherHost; import cydo.workflow.discovery.service : DiscoveryService, DiscoveryServiceHost, DiscoveryTaskSnapshot, ImportableReconciliationCommit, ImportableScanRecord, @@ -1263,9 +1264,9 @@ class App switch (json.type) { - case "create_task": handleCreateTaskMsg(ws, json); break; + case "create_task": withNormalizedImages(ws, json, (WsMessage m) { handleCreateTaskMsg(ws, m); }); break; case "request_history": handleRequestHistory(ws, json); break; - case "message": handleUserMessage(json); break; + case "message": withNormalizedImages(ws, json, (WsMessage m) { handleUserMessage(m); }); break; case "resume": handleResumeMsg(json); break; case "interrupt": handleInterruptMsg(json); break; case "sigint": handleSigintMsg(json); break; @@ -1458,6 +1459,35 @@ class App } } + /// Run a content-carrying handler with its image blocks made acceptable to + /// the agent APIs first. Synchronous when nothing needs converting, so the + /// common path keeps its ordering; a message whose photos need transcoding + /// is dispatched once the conversions finish, and is refused with a visible + /// error rather than forwarded if one of them cannot be converted. + private void withNormalizedImages(WebSocketAdapter ws, WsMessage json, + void delegate(WsMessage) handler) + { + import ae.utils.json : jsonParse, toJson, JSONFragment; + + if (json.content.json is null) + { + handler(json); + return; + } + auto blocks = jsonParse!(ContentBlock[])(json.content.json); + normalizeImageBlocks(blocks, (ImageNormalization normalized) { + if (normalized.error.length > 0) + { + import std.logger : warningf; + warningf("image attachment rejected for tid=%d: %s", json.tid, normalized.error); + ws.send(Data(toJson(ErrorMessage("error", normalized.error, json.tid)).representation)); + return; + } + json.content = JSONFragment(toJson(normalized.blocks)); + handler(json); + }); + } + private void handleRequestHistory(WebSocketAdapter ws, WsMessage json) { historyPipeline.handleRequestHistory(ws, json.tid); diff --git a/source/cydo/web/images.d b/source/cydo/web/images.d new file mode 100644 index 00000000..ee0bdb8b --- /dev/null +++ b/source/cydo/web/images.d @@ -0,0 +1,233 @@ +/// Image blocks arriving from a browser, normalized into what the agent APIs +/// accept before any handler sees them. +/// +/// The case that made this necessary: an iPhone camera photo is HEIC, and a +/// mobile browser hands it over as the original file. No agent API takes HEIC, +/// so the block is transcoded to JPEG here with ImageMagick. The conversion is +/// asynchronous: the event loop is single-threaded and shared by every client, +/// so blocking it for the second a transcode takes would stall everyone's +/// streaming. +module cydo.web.images; + +import std.base64 : Base64; +import std.process : Config, Pid, pipe, spawnProcess; +import std.stdio : File; + +import ae.net.asockets : DisconnectType, FileConnection; +import ae.sys.data : Data; +import ae.sys.process : asyncWait; +import ae.utils.array : asBytes; + +import cydo.protocol : ContentBlock; + +/// media types the agent APIs accept as-is +bool isNativeImageType(string mediaType) +{ + switch (mediaType) + { + case "image/jpeg", "image/png", "image/gif", "image/webp": return true; + default: return false; + } +} + +/// media types this module can turn into a native one +bool isConvertibleImageType(string mediaType) +{ + return mediaType == "image/heic" || mediaType == "image/heif"; +} + +/// Outcome of normalizing one message's blocks. +struct ImageNormalization +{ + ContentBlock[] blocks; + string error; /// non-empty when a block could not be made acceptable +} + +/// Rewrite any convertible image block in `blocks` and hand the result to +/// `done`. Calls `done` synchronously when nothing needs converting, so the +/// ordinary path keeps its ordering guarantees; otherwise `done` runs once +/// every conversion has finished. +void normalizeImageBlocks(ContentBlock[] blocks, void delegate(ImageNormalization) done) +{ + size_t pending = 0; + foreach (ref b; blocks) + if (b.type == "image" && isConvertibleImageType(b.media_type)) + pending++; + if (pending == 0) + { + done(ImageNormalization(blocks, null)); + return; + } + + auto result = ImageNormalization(blocks.dup, null); + size_t remaining = pending; + + // one frame per conversion: a delegate made in a loop body shares the + // function's single closure frame, so every one of them would otherwise + // see the final loop index and write into the same block. a parameter of + // a helper call is a fresh variable each time + void convertBlock(size_t index) + { + convertToJpeg(Base64.decode(result.blocks[index].data), (ubyte[] jpeg, string error) { + if (error.length > 0) + { + if (result.error.length == 0) + result.error = error; + } + else + { + result.blocks[index].data = Base64.encode(jpeg).idup; + result.blocks[index].media_type = "image/jpeg"; + } + if (--remaining == 0) + done(result); + }); + } + + foreach (i, ref b; result.blocks) + if (b.type == "image" && isConvertibleImageType(b.media_type)) + convertBlock(i); +} + +/// Transcode one image to JPEG with ImageMagick, asynchronously. +/// +/// `magick` is resolved through PATH at call time, so a missing install is an +/// error for the one attachment rather than a startup failure for the server. +void convertToJpeg(ubyte[] input, void delegate(ubyte[] jpeg, string error) done) +{ + import std.process : environment; + import std.file : exists; + import std.path : buildPath; + import std.algorithm : splitter; + + string magick; + foreach (dir; environment.get("PATH", "").splitter(':')) + { + auto candidate = buildPath(dir, "magick"); + if (candidate.exists) + { + magick = candidate; + break; + } + } + if (magick.length == 0) + { + done(null, "cannot convert this image: ImageMagick (magick) is not installed on the server"); + return; + } + + auto stdinPipe = pipe(); + auto stdoutPipe = pipe(); + auto stderrPipe = pipe(); + // quality 90 keeps a phone photo well under the agent APIs' size limits + // while staying visually lossless for their purposes + auto pid = spawnProcess( + [magick, "-", "-auto-orient", "-quality", "90", "jpeg:-"], + stdinPipe.readEnd, stdoutPipe.writeEnd, stderrPipe.writeEnd, + null, Config.none); + stdinPipe.readEnd.close(); + stdoutPipe.writeEnd.close(); + stderrPipe.writeEnd.close(); + + import core.sys.posix.unistd : dup; + auto stdinConn = new FileConnection(dup(stdinPipe.writeEnd.fileno)); + auto stdoutConn = new FileConnection(dup(stdoutPipe.readEnd.fileno)); + auto stderrConn = new FileConnection(dup(stderrPipe.readEnd.fileno)); + stdinPipe.writeEnd.close(); + stdoutPipe.readEnd.close(); + stderrPipe.readEnd.close(); + + ubyte[] output; + ubyte[] errors; + bool stdoutDone, stderrDone, exited; + int status; + bool finished; + + void finish() + { + if (finished || !stdoutDone || !stderrDone || !exited) + return; + finished = true; + if (status != 0 || output.length == 0) + { + import std.string : strip; + auto detail = (cast(string) errors).strip; + done(null, "cannot convert this image to JPEG" + ~ (detail.length ? ": " ~ detail : "")); + } + else + done(output, null); + } + + stdoutConn.handleReadData = (Data data) { output ~= cast(ubyte[]) data.toGC(); }; + stdoutConn.handleDisconnect = (string, DisconnectType) { stdoutDone = true; finish(); }; + stderrConn.handleReadData = (Data data) { errors ~= cast(ubyte[]) data.toGC(); }; + stderrConn.handleDisconnect = (string, DisconnectType) { stderrDone = true; finish(); }; + asyncWait(pid, (int code) { status = code; exited = true; finish(); }); + + // a requested disconnect on a connection with queued writes closes the fd + // only after everything has been flushed, which is what hands magick its EOF + stdinConn.send(Data(input)); + stdinConn.disconnect("input written"); +} + +unittest +{ + import std.process : environment, execute; + import ae.net.asockets : socketManager; + import ae.utils.json : JSONFragment; + + // needs a real ImageMagick with HEIC support; the nix sandbox has none, so + // the test proves the pipeline where it can and is silent where it cannot + auto probe = execute(["sh", "-c", "command -v magick >/dev/null 2>&1 && magick -list format | grep -q HEIC"]); + if (probe.status != 0) + return; + + auto heic = execute(["magick", "-size", "8x6", "xc:tomato", "heic:-"]); + assert(heic.status == 0 && heic.output.length > 12, "could not produce a HEIC fixture"); + + auto blocks = [ + ContentBlock("text", "caption"), + ContentBlock("image", null, null, null, JSONFragment.init, + Base64.encode(cast(ubyte[]) heic.output).idup, "image/heic"), + ]; + ImageNormalization got; + bool called; + normalizeImageBlocks(blocks, (ImageNormalization r) { got = r; called = true; }); + assert(!called, "a convertible block must not complete synchronously"); + while (!called) + socketManager.loop(); + assert(got.error.length == 0, got.error); + assert(got.blocks[0].type == "text" && got.blocks[0].text == "caption", "other blocks pass through untouched"); + assert(got.blocks[1].media_type == "image/jpeg"); + auto jpeg = Base64.decode(got.blocks[1].data); + assert(jpeg.length > 2 && jpeg[0] == 0xFF && jpeg[1] == 0xD8, "output must be a JPEG"); + + // two photos in one message: each must land in its own block. a single + // shared closure frame made both conversions write the second slot, so the + // first went to the agent still as HEIC + auto second = execute(["magick", "-size", "12x9", "xc:navy", "heic:-"]); + assert(second.status == 0); + auto pair = [ + ContentBlock("image", null, null, null, JSONFragment.init, + Base64.encode(cast(ubyte[]) heic.output).idup, "image/heic"), + ContentBlock("text", "between"), + ContentBlock("image", null, null, null, JSONFragment.init, + Base64.encode(cast(ubyte[]) second.output).idup, "image/heic"), + ]; + ImageNormalization both; + bool pairDone; + normalizeImageBlocks(pair, (ImageNormalization r) { both = r; pairDone = true; }); + while (!pairDone) + socketManager.loop(); + assert(both.error.length == 0, both.error); + assert(both.blocks[0].media_type == "image/jpeg" && both.blocks[2].media_type == "image/jpeg", + "every convertible block is converted, not just the last"); + assert(both.blocks[0].data != both.blocks[2].data, "each block keeps its own image"); + assert(both.blocks[1].text == "between"); + + // nothing convertible: the continuation runs synchronously, in order + bool sync; + normalizeImageBlocks([ContentBlock("text", "plain")], (ImageNormalization r) { sync = true; }); + assert(sync, "an unconvertible batch must complete synchronously"); +} diff --git a/web/src/components/InputBox.attach.test.tsx b/web/src/components/InputBox.attach.test.tsx new file mode 100644 index 00000000..b9add992 --- /dev/null +++ b/web/src/components/InputBox.attach.test.tsx @@ -0,0 +1,139 @@ +/** + * @vitest-environment jsdom + * @vitest-environment-options { "pretendToBeVisual": true } + * + * Image intake through the composer: every file handed over must either + * become a visible attachment or a visible refusal. A photo that vanishes + * with no trace is indistinguishable from a broken composer. + */ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { render } from "preact"; +import { act } from "preact/test-utils"; +import { InputBox, drafts } from "./InputBox"; + +vi.hoisted(() => { + vi.stubGlobal("CSS", { supports: () => false }); +}); + +const SESSION = "attach-session"; + +/** FileReader.readAsDataURL and the type sniffer resolve on tasks; let them land. */ +const flushAsync = () => new Promise((resolve) => setTimeout(resolve, 60)); + +async function mountComposer(container: HTMLElement) { + await act(() => { + render( + {}} + onInterrupt={() => {}} + isProcessing={false} + stdinClosed={false} + disabled={false} + sessionId={SESSION} + />, + container, + ); + }); +} + +/** A File with the leading bytes of the named format, typed as the browser + * would type it (an empty type is what Gecko assigns to .heic). */ +function imageFile(name: string, type: string, magic?: number[]) { + const bytes = magic ?? + { + "image/jpeg": [0xff, 0xd8, 0xff, 0xe0, 0, 0x10, 0x4a, 0x46], + "image/png": [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a], + }[type] ?? [1, 2, 3, 4]; + return new File([new Uint8Array(bytes)], name, { type }); +} + +/** bytes 4..12 of an iPhone camera HEIC: "ftypheic" */ +const HEIC_MAGIC = [ + 0, 0, 0, 0x18, 0x66, 0x74, 0x79, 0x70, 0x68, 0x65, 0x69, 0x63, +]; + +/** Deliver files the way a drag-and-drop does. */ +async function dropFiles(container: HTMLElement, files: File[]) { + const box = container.querySelector(".input-box"); + expect(box).not.toBeNull(); + // a FileList stand-in: indexed access, length, item(), and iteration + const list: Record = { + item: (index: number) => files[index] ?? null, + [Symbol.iterator]: files[Symbol.iterator].bind(files), + }; + files.forEach((file, index) => { + list[index] = file; + }); + list.length = files.length; + const event = new Event("drop", { bubbles: true, cancelable: true }); + Object.defineProperty(event, "dataTransfer", { + value: { files: list, types: ["Files"] }, + }); + await act(() => { + box!.dispatchEvent(event); + }); +} + +describe("composer image attachment", () => { + let container: HTMLElement; + + beforeEach(() => { + drafts.clear(); + container = document.createElement("div"); + document.body.appendChild(container); + }); + + it("attaches dropped images as previews", async () => { + await mountComposer(container); + await dropFiles(container, [ + imageFile("one.jpg", "image/jpeg"), + imageFile("two.png", "image/png"), + ]); + await act(async () => { + await flushAsync(); + }); + + expect(container.querySelectorAll(".image-preview img").length).toBe(2); + }); + + it("refuses image types nothing downstream can use, and says so", async () => { + await mountComposer(container); + await dropFiles(container, [imageFile("scan.bmp", "image/bmp")]); + await act(async () => { + await flushAsync(); + }); + + expect(container.querySelectorAll(".image-preview").length).toBe(0); + const error = container.querySelector(".attach-error"); + expect(error).not.toBeNull(); + expect(error!.textContent).toContain("image/bmp"); + }); + + it("recognizes a HEIC photo by its bytes even with no browser-assigned type", async () => { + await mountComposer(container); + // gecko has no mime entry for .heic, so the File arrives with type "" + await dropFiles(container, [imageFile("IMG_0001.HEIC", "", HEIC_MAGIC)]); + await act(async () => { + await flushAsync(); + }); + + // nothing silently dropped: a chip stands in for the undecodable preview + expect(container.querySelector(".attach-error")).toBeNull(); + const label = container.querySelector(".image-preview-label"); + expect(label).not.toBeNull(); + expect(label!.textContent).toContain("HEIC"); + expect(container.querySelectorAll(".image-preview img").length).toBe(0); + }); + + it("names an unrecognized file instead of dropping it silently", async () => { + await mountComposer(container); + await dropFiles(container, [imageFile("mystery.bin", "", [0, 1, 2, 3])]); + await act(async () => { + await flushAsync(); + }); + + const error = container.querySelector(".attach-error"); + expect(error).not.toBeNull(); + expect(error!.textContent).toContain("mystery.bin"); + }); +}); diff --git a/web/src/components/InputBox.tsx b/web/src/components/InputBox.tsx index 636f49e9..83196e5e 100644 --- a/web/src/components/InputBox.tsx +++ b/web/src/components/InputBox.tsx @@ -132,6 +132,45 @@ function publishImages(entry: ControlledImageEntry, images: ImageAttachment[]) { } } +// what the agent APIs accept as-is, plus what the server transcodes for them: +// an iPhone camera photo is HEIC, and a mobile browser hands over the original +const SUPPORTED_IMAGE_TYPES = new Set([ + "image/jpeg", + "image/png", + "image/gif", + "image/webp", + "image/heic", + "image/heif", +]); + +// formats the browser itself cannot decode, so the preview shows a label +// instead of a broken ; the server turns them into JPEG before sending +const PREVIEW_UNDECODABLE = new Set(["image/heic", "image/heif"]); + +/** The real format from the file's leading bytes. + * + * `File.type` comes from the browser's extension table, and a dragged or + * pasted file can arrive with no usable extension or one the browser has no + * entry for (Gecko has none for .heic), which leaves `type` empty. The bytes + * do not lie. + */ +async function sniffImageType(file: File): Promise { + const head = new Uint8Array(await file.slice(0, 16).arrayBuffer()); + const ascii = (from: number, to: number) => + String.fromCharCode(...head.subarray(from, to)); + if (head[0] === 0xff && head[1] === 0xd8 && head[2] === 0xff) + return "image/jpeg"; + if (head[0] === 0x89 && ascii(1, 4) === "PNG") return "image/png"; + if (ascii(0, 4) === "GIF8") return "image/gif"; + if (ascii(0, 4) === "RIFF" && ascii(8, 12) === "WEBP") return "image/webp"; + if (ascii(4, 8) === "ftyp") { + const brand = ascii(8, 12); + if (["heic", "heix", "hevc", "hevx", "mif1", "msf1"].includes(brand)) + return "image/heic"; + } + return file.type; +} + function useImageAttachments( enabled = true, resetToken?: number, @@ -151,6 +190,7 @@ function useImageAttachments( () => imageEntry?.generation ?? 0, ); const [isDragging, setIsDragging] = useState(false); + const [attachError, setAttachError] = useState(null); const enabledRef = useRef(enabled); const resetTokenRef = useRef(resetToken); const generationRef = useRef(imageEntry?.generation ?? 0); @@ -216,7 +256,30 @@ function useImageAttachments( }; const processFile = (file: File) => { - if (!enabledRef.current || !file.type.startsWith("image/")) return; + if (!enabledRef.current) return; + // a type the browser resolved from a known extension is trustworthy and + // keeps the synchronous path; sniffing is for the files it could not type + if (SUPPORTED_IMAGE_TYPES.has(file.type)) { + attachTyped(file, file.type); + return; + } + void sniffImageType(file).then((mediaType) => { + attachTyped(file, mediaType); + }); + }; + + // nothing here may drop a file silently: a picked photo that vanishes with no + // trace is indistinguishable from a broken attach control + const attachTyped = (file: File, mediaType: string) => { + if (!enabledRef.current) return; + if (!SUPPORTED_IMAGE_TYPES.has(mediaType)) { + const label = mediaType || `"${file.name}" (unrecognized format)`; + setAttachError( + `${label} can't be attached; JPEG, PNG, GIF, WebP and HEIC work`, + ); + return; + } + setAttachError(null); const generation = generationRef.current; const entry = imageEntry; const reader = new FileReader(); @@ -227,7 +290,7 @@ function useImageAttachments( id: crypto.randomUUID(), dataURL, base64, - mediaType: file.type, + mediaType, }; if (entry) { if ( @@ -297,6 +360,10 @@ function useImageAttachments( images: enabled && imagesGeneration === generationRef.current ? images : [], setImages, isDragging: enabled && isDragging, + attachError, + dismissAttachError: () => { + setAttachError(null); + }, onPaste, onDragOver, onDragLeave, @@ -304,6 +371,20 @@ function useImageAttachments( }; } +function AttachError({ + message, + onDismiss, +}: { + message: string; + onDismiss: () => void; +}) { + return ( +
+ {message} +
+ ); +} + function ImagePreviews({ images, onRemove, @@ -318,7 +399,13 @@ function ImagePreviews({
{images.map((image) => (
- Attached + {PREVIEW_UNDECODABLE.has(image.mediaType) ? ( +
+ {image.mediaType.replace("image/", "").toUpperCase()} photo +
+ ) : ( + Attached + )}