diff --git a/.changeset/server-function-uploads.md b/.changeset/server-function-uploads.md new file mode 100644 index 000000000..3a68bdd57 --- /dev/null +++ b/.changeset/server-function-uploads.md @@ -0,0 +1,5 @@ +--- +"@solidjs/start": minor +--- + +Let server functions take files and blobs anywhere in their arguments, not only as the only argument. A call like `upload(file, { tags })`, or one with files in arrays, objects, maps, sets or form data, sends them as parts of multipart form data next to the serialized arguments, and the server function gets them as `File` values. Calls without files are sent as before. diff --git a/apps/tests/src/e2e/server-function.test.ts b/apps/tests/src/e2e/server-function.test.ts index abb9861f7..d668deaa6 100644 --- a/apps/tests/src/e2e/server-function.test.ts +++ b/apps/tests/src/e2e/server-function.test.ts @@ -94,6 +94,18 @@ test.describe("server-function", () => { await page.goto("http://localhost:3000/server-function-blob"); await expect(page.locator("#server-fn-test")).toContainText('{"result":true}'); }); + test("should send files next to other arguments", async ({ page }) => { + await page.goto("http://localhost:3000/server-function-upload"); + await expect(page.locator("#server-fn-test")).toContainText( + JSON.stringify({ + name: "hello-world.txt", + text: "Hello, World!", + tags: ["a", "b"], + attachments: ["one", "two"], + avatar: "avatar.png:png", + }), + ); + }); test("should remove exports for non-function values when top-level use server is used", async ({ page, }) => { diff --git a/apps/tests/src/routes/server-function-upload.tsx b/apps/tests/src/routes/server-function-upload.tsx new file mode 100644 index 000000000..874d3047b --- /dev/null +++ b/apps/tests/src/routes/server-function-upload.tsx @@ -0,0 +1,36 @@ +import { createEffect, createSignal } from "solid-js"; + +async function upload(file: File, meta: { tags: string[]; attachments: Blob[] }, form: FormData) { + "use server"; + const attachments = await Promise.all(meta.attachments.map(blob => blob.text())); + const avatar = form.get("avatar") as File; + return { + name: file.name, + text: await file.text(), + tags: meta.tags, + attachments, + avatar: `${avatar.name}:${await avatar.text()}`, + }; +} + +export default function App() { + const [output, setOutput] = createSignal(); + + createEffect(async () => { + const form = new FormData(); + form.set("avatar", new File(["png"], "avatar.png", { type: "image/png" })); + setOutput( + await upload( + new File(["Hello, World!"], "hello-world.txt"), + { tags: ["a", "b"], attachments: [new Blob(["one"]), new Blob(["two"])] }, + form, + ), + ); + }); + + return ( +
+ {JSON.stringify(output())} +
+ ); +} diff --git a/packages/start/src/fns/client.ts b/packages/start/src/fns/client.ts index 1a8a68ce0..9f74c50cb 100644 --- a/packages/start/src/fns/client.ts +++ b/packages/start/src/fns/client.ts @@ -5,6 +5,7 @@ import { serializeToJSONString, } from "./serialization.ts"; import { BODY_FORMAT_KEY, BodyFormat, extractBody, getHeadersAndBody } from "./shared.ts"; +import { appendUploads, ARGS_PART, createUploadSerializer } from "./uploads.ts"; let INSTANCE = 0; @@ -55,10 +56,26 @@ async function initializeResponse( } } // Fallback to seroval + const files: Blob[] = []; + // TODO(Alexis): move to serializeToJSONStream + const serialized = await serializeToJSONString(args, [createUploadSerializer(files)]); + if (files.length > 0) { + // Files go as form parts, so they are sent as they are instead of encoded. + const formData = new FormData(); + formData.set(ARGS_PART, serialized); + await appendUploads(formData, files); + return createRequest(base, id, instance, { + ...options, + body: formData, + headers: { + ...options.headers, + [BODY_FORMAT_KEY]: BodyFormat.SerovalFormData, + }, + }); + } return createRequest(base, id, instance, { ...options, - // TODO(Alexis): move to serializeToJSONStream - body: await serializeToJSONString(args), + body: serialized, // duplex: 'half', // body: serializeToJSONStream(args), headers: { diff --git a/packages/start/src/fns/handler.ts b/packages/start/src/fns/handler.ts index cf0a7efd4..56b4c5748 100644 --- a/packages/start/src/fns/handler.ts +++ b/packages/start/src/fns/handler.ts @@ -184,7 +184,7 @@ async function runServerFunction(h3Event: H3Event) { if (request.method === "POST" && request.body !== null) { const bodyFormat = request.headers.get(BODY_FORMAT_KEY); const decoded = await extractBody("", false, request.clone()); - if (bodyFormat === BodyFormat.Seroval) { + if (bodyFormat === BodyFormat.Seroval || bodyFormat === BodyFormat.SerovalFormData) { parsed = decoded as any[]; } else { parsed.push(decoded); diff --git a/packages/start/src/fns/serialization.ts b/packages/start/src/fns/serialization.ts index a68a3b6d0..0ff67e4a5 100644 --- a/packages/start/src/fns/serialization.ts +++ b/packages/start/src/fns/serialization.ts @@ -5,6 +5,7 @@ import { fromCrossJSON, getCrossReferenceHeader, isStream, + type Plugin, type SerovalNode, toCrossJSONStream, } from "seroval"; @@ -47,6 +48,14 @@ const DEFAULT_PLUGINS = [ * rather than a runtime option. */ const PLUGINS = [...DEFAULT_PLUGINS, ...userPlugins]; + +/** + * Plugins for one call, such as the ones that carry files as form parts. They + * come first, so the values they handle never reach the other plugins. + */ +function withPlugins(extra: Plugin[] | undefined) { + return extra?.length ? [...extra, ...PLUGINS] : PLUGINS; +} const MAX_SERIALIZATION_DEPTH_LIMIT = 64; const DISABLED_FEATURES = Feature.RegExp; @@ -144,13 +153,13 @@ export function serializeToJSStream(id: string, value: any) { }); } -export function serializeToJSONStream(value: any) { +export function serializeToJSONStream(value: any, plugins?: Plugin[]) { return new ReadableStream({ start(controller) { toCrossJSONStream(value, { disabledFeatures: SERIALIZE_DISABLED_FEATURES, depthLimit: MAX_SERIALIZATION_DEPTH_LIMIT, - plugins: PLUGINS, + plugins: withPlugins(plugins), onParse(node) { enqueueChunk(controller, JSON.stringify(node)); }, @@ -197,6 +206,11 @@ export interface SerovalChunkReaderOptions { maxChunkSize?: number; } +export interface DeserializeJSONStreamOptions extends SerovalChunkReaderOptions { + /** Plugins for this body only, tried before the others. */ + plugins?: Plugin[]; +} + export class SerovalChunkReader { reader: ReadableStreamDefaultReader; done = false; @@ -368,19 +382,22 @@ function createEndOfStreamError() { return new Error("Server function stream ended unexpectedly."); } -export async function serializeToJSONString(value: any) { - const response = new Response(serializeToJSONStream(value)); +export async function serializeToJSONString(value: any, plugins?: Plugin[]) { + const response = new Response(serializeToJSONStream(value, plugins)); return await response.text(); } -export async function deserializeFromJSONString(json: string) { +export async function deserializeFromJSONString( + json: string, + options?: DeserializeJSONStreamOptions, +) { const blob = new Response(json); - return await deserializeJSONStream(blob); + return await deserializeJSONStream(blob, options); } export async function deserializeJSONStream( response: Response | Request, - options?: SerovalChunkReaderOptions, + options?: DeserializeJSONStreamOptions, ) { if (!response.body) { throw new Error("missing body"); @@ -392,13 +409,14 @@ export async function deserializeJSONStream( }); if (!result.done) { const refs = new DecodedRefs(); + const plugins = withPlugins(options?.plugins); function interpretChunk(chunk: string): unknown { const value = fromCrossJSON(JSON.parse(chunk) as SerovalNode, { refs, disabledFeatures: DISABLED_FEATURES, depthLimit: MAX_SERIALIZATION_DEPTH_LIMIT, - plugins: PLUGINS, + plugins, }); return value; } diff --git a/packages/start/src/fns/shared.ts b/packages/start/src/fns/shared.ts index e0a1d7c2e..7f1045bae 100644 --- a/packages/start/src/fns/shared.ts +++ b/packages/start/src/fns/shared.ts @@ -1,8 +1,10 @@ import { + deserializeFromJSONString, deserializeJSONStream, deserializeJSStream, MAX_REQUEST_CHUNK_SIZE, } from "./serialization.ts"; +import { ARGS_PART, createUploadDeserializer } from "./uploads.ts"; export const BODY_FORMAT_KEY = "X-Start-Type"; @@ -17,6 +19,8 @@ export const enum BodyFormat { File = "5", ArrayBuffer = "6", Uint8Array = "7", + /** Serialized arguments with their files as parts of multipart form data. */ + SerovalFormData = "8", } export function getHeadersAndBody(body: any): @@ -100,6 +104,18 @@ export async function extractBody(instance: string, client: boolean, source: Req clone, client ? undefined : { maxChunkSize: MAX_REQUEST_CHUNK_SIZE }, ); + case startType === BodyFormat.SerovalFormData: { + // Only a client sends files, so this is always read on the server. + const formData = await clone.formData(); + const args = formData.get(ARGS_PART); + if (typeof args !== "string") { + throw new Error("Malformed server function upload."); + } + return await deserializeFromJSONString(args, { + maxChunkSize: MAX_REQUEST_CHUNK_SIZE, + plugins: [createUploadDeserializer(formData)], + }); + } case startType === BodyFormat.String: return await clone.text(); case startType === BodyFormat.File: { diff --git a/packages/start/src/fns/uploads.spec.ts b/packages/start/src/fns/uploads.spec.ts new file mode 100644 index 000000000..61e46a514 --- /dev/null +++ b/packages/start/src/fns/uploads.spec.ts @@ -0,0 +1,165 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { deserializeFromJSONString, serializeToJSONString } from "./serialization.ts"; +import { BODY_FORMAT_KEY, BodyFormat, extractBody } from "./shared.ts"; +import { ARGS_PART, createUploadSerializer, FILE_PART } from "./uploads.ts"; + +vi.mock("../shared/dev-toolbar/functions/tracker.ts", () => ({ + pushRequest: vi.fn(), + pushResponse: vi.fn(), +})); + +const { cloneServerReference } = await import("./client.ts"); + +let sent: Request | undefined; + +beforeEach(() => { + sent = undefined; + vi.stubEnv("SERVER_BASE_URL", "http://localhost/"); + vi.stubGlobal( + "fetch", + vi.fn(async (request: Request) => { + sent = request; + return new Response(null, { status: 204 }); + }), + ); +}); + +afterEach(() => { + vi.unstubAllEnvs(); + vi.unstubAllGlobals(); +}); + +/** Calls a server function and returns the arguments the server reads from the request. */ +async function roundTrip(...args: unknown[]) { + await (cloneServerReference("upload-fn") as unknown as (...args: unknown[]) => Promise)( + ...args, + ); + return (await extractBody("", false, sent!)) as unknown[]; +} + +function uploadRequest(form: FormData) { + return new Request("http://localhost/_server", { + method: "POST", + body: form, + headers: { [BODY_FORMAT_KEY]: BodyFormat.SerovalFormData }, + }); +} + +describe("server function uploads", () => { + it("sends files next to other arguments as form parts", async () => { + const file = new File(["hello"], "hello.txt", { type: "text/plain" }); + + const [received, meta] = await roundTrip(file, { tags: ["a"] }); + + expect(sent!.headers.get(BODY_FORMAT_KEY)).toBe(BodyFormat.SerovalFormData); + expect(sent!.headers.get("content-type")).toMatch(/^multipart\/form-data/); + expect(received).toBeInstanceOf(File); + expect((received as File).name).toBe("hello.txt"); + expect((received as File).type).toBe("text/plain"); + expect(await (received as File).text()).toBe("hello"); + expect(meta).toEqual({ tags: ["a"] }); + }); + + it("finds files anywhere in the arguments", async () => { + const a = new File(["a"], "a.txt"); + const b = new File(["b"], "b.txt"); + const c = new File(["c"], "c.txt"); + + const [value] = (await roundTrip({ + list: [a], + map: new Map([["b", b]]), + set: new Set([c]), + })) as [{ list: File[]; map: Map; set: Set }]; + + expect(await value.list[0]!.text()).toBe("a"); + expect(await value.map.get("b")!.text()).toBe("b"); + expect(await [...value.set][0]!.text()).toBe("c"); + }); + + it("sends the files of form data that is not the only argument", async () => { + const form = new FormData(); + form.set("name", "avatar"); + form.set("file", new File(["png"], "avatar.png", { type: "image/png" })); + + const [received, count] = (await roundTrip(form, 1)) as [FormData, number]; + + expect(received).toBeInstanceOf(FormData); + expect(received.get("name")).toBe("avatar"); + expect(await (received.get("file") as File).text()).toBe("png"); + expect(count).toBe(1); + }); + + it("sends a file used twice once, and keeps it the same value", async () => { + const file = new File(["once"], "once.txt"); + + const [first, second] = await roundTrip(file, { again: file }); + + const form = await sent!.clone().formData(); + expect(form.has(`${FILE_PART}0`)).toBe(true); + expect(form.has(`${FILE_PART}1`)).toBe(false); + expect((second as { again: File }).again).toBe(first); + }); + + it("keeps the type of a blob", async () => { + const [received] = await roundTrip(new Blob(["{}"], { type: "application/json" }), 2); + + expect(received).toBeInstanceOf(Blob); + expect((received as Blob).type).toBe("application/json"); + expect(await (received as Blob).text()).toBe("{}"); + }); + + it("copies a blob from another frame", async () => { + const source = new Blob(["other"], { type: "text/plain" }); + // Not an instance of this realm's Blob, as one from another frame isn't. + const foreign = { + size: source.size, + type: source.type, + arrayBuffer: () => source.arrayBuffer(), + slice: () => source.slice(), + }; + + const [received] = await roundTrip(foreign, 1); + + expect(await (received as Blob).text()).toBe("other"); + }); + + it("keeps calls without files as serialized text", async () => { + await roundTrip({ a: 1 }, 2); + + expect(sent!.headers.get(BODY_FORMAT_KEY)).toBe(BodyFormat.Seroval); + expect(sent!.headers.get("content-type")).toBe("text/plain"); + }); + + it("rejects an upload whose part is missing", async () => { + const form = new FormData(); + const files: Blob[] = []; + form.set( + ARGS_PART, + await serializeToJSONString([new File(["x"], "x.txt")], [createUploadSerializer(files)]), + ); + + await expect(extractBody("", false, uploadRequest(form))).rejects.toThrow( + "Malformed server function upload.", + ); + }); + + it("rejects an upload without its arguments", async () => { + const form = new FormData(); + form.set(`${FILE_PART}0`, new File(["x"], "x.txt")); + + await expect(extractBody("", false, uploadRequest(form))).rejects.toThrow( + "Malformed server function upload.", + ); + }); + + it("does not read upload indexes from a body that is not multipart", async () => { + const body = await serializeToJSONString( + [new File(["x"], "x.txt")], + [createUploadSerializer([])], + ); + + // A plain body is read without the upload plugin, so the index has no meaning there. + await expect(deserializeFromJSONString(body)).rejects.toThrow(); + }); +}); diff --git a/packages/start/src/fns/uploads.ts b/packages/start/src/fns/uploads.ts new file mode 100644 index 000000000..9bc42f32a --- /dev/null +++ b/packages/start/src/fns/uploads.ts @@ -0,0 +1,141 @@ +import { createPlugin, type Plugin, type SerovalNode } from "seroval"; + +/** + * Files and blobs in the arguments of a server function call go as parts of + * multipart form data instead of inside the serialized arguments. The + * arguments go in the `ARGS_PART` field, and each file in `FILE_PART`, + * where the arguments hold only its index. + */ +export const ARGS_PART = "args"; +export const FILE_PART = "file"; + +const UPLOAD_TAG = "solid-start/upload"; + +/* + * Migration note: seroval's binary mode (`binary.serialize` and + * `binary.deserialize`, in development on seroval's `binary-mode` branch) + * could replace this when server function bodies move to it. + * + * Binary mode would be better for the arguments in general: + * - one body and one format for every value, instead of JSON in a form field; + * - `ArrayBuffer` and typed array bytes sent as they are, with no base64 and + * no `maxBase64Length` limit; + * - decoding that validates untrusted input and runs no code, as `fromCrossJSON` + * does today. + * + * For files, the form parts here are still better: + * - `BlobPlugin` and `FilePlugin` read the whole file with `arrayBuffer()` + * before sending it, so a large upload is held in memory on the client. A + * `File` in `FormData` is read by the browser as it sends the request. + * - Sending binary mode output as it is produced needs a streamed request + * body (`duplex: "half"`), which not every browser supports. Otherwise the + * chunks are collected first, which holds the file in memory again. + * - Multipart bodies work in every runtime and with every proxy, and servers + * read them with `request.formData()`. + * + * Neither limits the total size: binary mode leaves that to the transport, + * and `request.formData()` reads the whole body. A limit belongs in the + * server either way. + * + * When migrating, the arguments can move to binary mode while files keep + * going as form parts, with binary mode's plugin section writing the part + * index the way `createUploadSerializer` does. A server should accept both + * bodies for a while, because an open tab can call a newer server. + */ + +type UploadNode = { i: SerovalNode }; + +/** + * True for a file or a blob. One from another frame is an instance of that + * frame's `Blob`, so its shape tells instead. + */ +function isBlob(value: unknown): value is Blob { + if (value instanceof Blob) return true; + const blob = value as Partial | null; + return ( + typeof blob === "object" && + blob !== null && + typeof blob.size === "number" && + typeof blob.type === "string" && + typeof blob.arrayBuffer === "function" && + typeof blob.slice === "function" + ); +} + +function malformed(): Error { + return new Error("Malformed server function upload."); +} + +/** + * Serializes each file or blob in the arguments as its index in `files`, so + * the call can send it as a form part. Seroval calls this wherever it finds + * one: in arrays, objects, maps, sets and form data. + */ +export function createUploadSerializer(files: Blob[]): Plugin { + const parse = (value: Blob) => files.push(value) - 1; + return createPlugin({ + tag: UPLOAD_TAG, + test: isBlob, + parse: { + sync: (value, ctx) => ({ i: ctx.parse(parse(value)) }), + async: async (value, ctx) => ({ i: await ctx.parse(parse(value)) }), + stream: (value, ctx) => ({ i: ctx.parse(parse(value)) }), + }, + serialize: (node, ctx) => ctx.serialize(node.i), + deserialize: () => { + throw malformed(); + }, + }); +} + +/** + * Turns each index written by `createUploadSerializer` back into its part of + * `form`. It is only used for a multipart call, so an index in any other body + * fails to deserialize. + */ +export function createUploadDeserializer(form: FormData): Plugin { + return createPlugin({ + tag: UPLOAD_TAG, + test: () => false, + parse: { + sync: () => { + throw malformed(); + }, + async: async () => { + throw malformed(); + }, + stream: () => { + throw malformed(); + }, + }, + serialize: () => { + throw malformed(); + }, + deserialize: (node, ctx) => { + const index = ctx.deserialize(node.i); + if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0) { + throw malformed(); + } + const part = form.get(`${FILE_PART}${index}`); + if (!(part instanceof Blob)) throw malformed(); + return part; + }, + }); +} + +/** + * A blob of another frame is copied, because form data only takes blobs of + * its own. + */ +export async function appendUploads(form: FormData, files: Blob[]): Promise { + const blobs = await Promise.all( + files.map(async file => + file instanceof Blob + ? file + : new Blob([await (file as Blob).arrayBuffer()], { type: (file as Blob).type }), + ), + ); + for (const [index, blob] of blobs.entries()) { + form.set(`${FILE_PART}${index}`, blob, (files[index] as Partial).name ?? "blob"); + } +}