diff --git a/.changeset/server-function-uploads.md b/.changeset/server-function-uploads.md new file mode 100644 index 000000000..685502579 --- /dev/null +++ b/.changeset/server-function-uploads.md @@ -0,0 +1,5 @@ +--- +"@solidjs/start": minor +--- + +Let server functions take files and blobs anywhere in their arguments, not only inside a `FormData` that is the only argument. A call like `upload(file, { tags })`, or one with files in arrays, objects, maps, sets or form data, sends them as parts of multipart form data next to the serialized arguments, and the server function gets them as `File` values. Calls without files are sent as before. diff --git a/apps/tests/cypress/e2e/server-function.cy.ts b/apps/tests/cypress/e2e/server-function.cy.ts index 903758fbe..f1c575c2c 100644 --- a/apps/tests/cypress/e2e/server-function.cy.ts +++ b/apps/tests/cypress/e2e/server-function.cy.ts @@ -20,6 +20,19 @@ describe("server-function", () => { cy.get("#server-fn-test").contains('{"serverFnWithNpmModule":[2,4,6]}'); }); + it("should send files next to other arguments", () => { + cy.visit("/server-function-upload"); + cy.get("#server-fn-test").contains( + JSON.stringify({ + name: "hello-world.txt", + text: "Hello, World!", + tags: ["a", "b"], + attachments: ["one", "two"], + avatar: "avatar.png:png" + }) + ); + }); + it("should have isServer true in the server function - toplevel", () => { cy.visit("/is-server-toplevel"); cy.get("#server-fn-test").contains('{"serverFnWithIsServer":true}'); diff --git a/apps/tests/src/routes/server-function-upload.tsx b/apps/tests/src/routes/server-function-upload.tsx new file mode 100644 index 000000000..6e34a6b0e --- /dev/null +++ b/apps/tests/src/routes/server-function-upload.tsx @@ -0,0 +1,36 @@ +import { createEffect, createSignal } from "solid-js"; + +async function upload(file: File, meta: { tags: string[]; attachments: Blob[] }, form: FormData) { + "use server"; + const attachments = await Promise.all(meta.attachments.map(blob => blob.text())); + const avatar = form.get("avatar") as File; + return { + name: file.name, + text: await file.text(), + tags: meta.tags, + attachments, + avatar: `${avatar.name}:${await avatar.text()}` + }; +} + +export default function App() { + const [output, setOutput] = createSignal(); + + createEffect(async () => { + const form = new FormData(); + form.set("avatar", new File(["png"], "avatar.png", { type: "image/png" })); + setOutput( + await upload( + new File(["Hello, World!"], "hello-world.txt"), + { tags: ["a", "b"], attachments: [new Blob(["one"]), new Blob(["two"])] }, + form + ) + ); + }); + + return ( +
+ {JSON.stringify(output())} +
+ ); +} diff --git a/packages/start/src/runtime/serialization.ts b/packages/start/src/runtime/serialization.ts index c3bd411fb..6c7221c43 100644 --- a/packages/start/src/runtime/serialization.ts +++ b/packages/start/src/runtime/serialization.ts @@ -5,6 +5,7 @@ import { fromCrossJSON, getCrossReferenceHeader, isStream, + type Plugin, type SerovalJSON, type SerovalNode, toCrossJSONStream, @@ -69,6 +70,14 @@ function createChunk(data: string): Uint8Array { return chunk; } +/** + * Plugins for one call, such as the ones that carry files as form parts. They + * come first, so the values they handle never reach the other plugins. + */ +function withPlugins(extra: Plugin[] | undefined) { + return extra?.length ? [...extra, ...DEFAULT_PLUGINS] : DEFAULT_PLUGINS; +} + export function serializeToJSStream(id: string, value: any) { return new ReadableStream({ start(controller) { @@ -204,11 +213,11 @@ class SerovalChunkReader { } } -export async function serializeToJSONString(value: any) { +export async function serializeToJSONString(value: any, plugins?: Plugin[]) { // const response = new Response(serializeToJSONStream(value)); // return await response.text(); return JSON.stringify(await toJSONAsync(value, { - plugins: DEFAULT_PLUGINS, + plugins: withPlugins(plugins), depthLimit: MAX_SERIALIZATION_DEPTH_LIMIT, disabledFeatures: DISABLED_FEATURES, })); @@ -275,14 +284,14 @@ function createEndOfStreamError() { * is used because it exposes its refs, and it reads the same nodes `toJSONAsync` * emits. */ -export async function deserializeFromJSONString(json: string) { +export async function deserializeFromJSONString(json: string, plugins?: Plugin[]) { const source = JSON.parse(json) as SerovalJSON; const refs = new DecodedRefs(); let value: unknown; try { value = fromCrossJSON(source.t, { refs, - plugins: DEFAULT_PLUGINS, + plugins: withPlugins(plugins), disabledFeatures: DISABLED_FEATURES, }); } catch (error) { diff --git a/packages/start/src/runtime/server-handler.ts b/packages/start/src/runtime/server-handler.ts index b6a91fef9..c0ba63728 100644 --- a/packages/start/src/runtime/server-handler.ts +++ b/packages/start/src/runtime/server-handler.ts @@ -21,6 +21,7 @@ import serverFnManifest from "solidstart:server-fn-manifest"; import { setThrownCacheControl, withDefaultCacheControl } from "./cache-control"; import { isCrossSiteRequest } from "./cross-site"; import { deserializeFromJSONString, serializeToJSONStream, serializeToJSStream } from "./serialization"; +import { readUploadArgs, UPLOAD_HEADER } from "./uploads"; async function handleServerFunction(h3Event: HTTPEvent) { let response: Awaited>; @@ -120,7 +121,9 @@ async function runServerFunction(h3Event: HTTPEvent) { const tmpReq = isH3EventBodyStreamLocked ? request : new Request(request, { ...request, body: requestBody }); - if (request.headers.get('x-serialized')) { + if (request.headers.get(UPLOAD_HEADER)) { + parsed = (await readUploadArgs(await tmpReq.formData())) as any[]; + } else if (request.headers.get('x-serialized')) { parsed = await deserializeFromJSONString(await tmpReq.text()) as any[]; } else if ( contentType?.startsWith("multipart/form-data") || diff --git a/packages/start/src/runtime/server-runtime.ts b/packages/start/src/runtime/server-runtime.ts index 003ae02c7..8e2e856d9 100644 --- a/packages/start/src/runtime/server-runtime.ts +++ b/packages/start/src/runtime/server-runtime.ts @@ -5,6 +5,7 @@ import { deserializeJSStream, serializeToJSONString, } from "./serialization"; +import { appendUploads, ARGS_PART, createUploadSerializer, UPLOAD_HEADER } from "./uploads"; let INSTANCE = 0; @@ -25,6 +26,42 @@ function createRequest( }); } +async function createSerializedRequest( + base: string, + id: string, + instance: string, + options: RequestInit, + args: any[], +) { + const files: Blob[] = []; + const serialized = await serializeToJSONString(args, [createUploadSerializer(files)]); + if (files.length > 0) { + // Files go as form parts, so they are sent as they are instead of encoded. + const formData = new FormData(); + formData.set(ARGS_PART, serialized); + await appendUploads(formData, files); + return createRequest(base, id, instance, { + ...options, + body: formData, + headers: { + ...options.headers, + [UPLOAD_HEADER]: "true", + }, + }); + } + return createRequest(base, id, instance, { + ...options, + body: serialized, + // duplex: 'half', + // body: serializeToJSONStream(args), + headers: { + ...options.headers, + "x-serialized": "true", + "Content-Type": "text/plain" + }, + }); +} + async function fetchServerFunction( base: string, id: string, @@ -45,17 +82,7 @@ async function fetchServerFunction( "Content-Type": "application/x-www-form-urlencoded", }, }) - : createRequest(base, id, instance, { - ...options, - body: await serializeToJSONString(args), - // duplex: 'half', - // body: serializeToJSONStream(args), - headers: { - ...options.headers, - "x-serialized": "true", - "Content-Type": "text/plain" - }, - })); + : createSerializedRequest(base, id, instance, options, args)); if ( response.headers.has("Location") || diff --git a/packages/start/src/runtime/uploads.test.ts b/packages/start/src/runtime/uploads.test.ts new file mode 100644 index 000000000..e8c85c475 --- /dev/null +++ b/packages/start/src/runtime/uploads.test.ts @@ -0,0 +1,149 @@ +import { createApp, toWebHandler } from "vinxi/http"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { serializeToJSONString } from "./serialization"; +import { + ARGS_PART, + createUploadSerializer, + FILE_PART, + readUploadArgs, + UPLOAD_HEADER +} from "./uploads"; + +const serverFn = vi.hoisted(() => ({ + args: [] as unknown[] +})); + +vi.mock("solidstart:server-fn-manifest", () => ({ + default: { + fn: { + functionName: "fn", + importer: async () => ({ + fn: (...args: unknown[]) => { + serverFn.args = args; + return "done"; + } + }) + } + } +})); +vi.mock("../server/handler", () => ({ getExpectedRedirectStatus: () => 302 })); +vi.mock("../server/pageEvent", () => ({ createPageEvent: async () => {} })); + +const { default: serverHandler } = await import("./server-handler"); +const { createServerReference } = await import("./server-runtime"); +const handle = toWebHandler(createApp().use(serverHandler)); + +let sent: Request | undefined; + +beforeEach(() => { + serverFn.args = []; + sent = undefined; + vi.stubEnv("SERVER_BASE_URL", "http://localhost:3000"); + vi.stubGlobal("fetch", async (url: string, init: RequestInit) => { + sent = new Request(url, init); + return handle(sent.clone()); + }); +}); + +/** Calls the server function through the client, and returns what it received. */ +async function roundTrip(...args: unknown[]) { + const fn = createServerReference(() => {}, "fn", "fn") as unknown as ( + ...args: unknown[] + ) => Promise; + expect(await fn(...args)).toBe("done"); + return serverFn.args; +} + +describe("server function uploads", () => { + it("sends a file as the only argument", async () => { + const [received] = await roundTrip(new File(["hello"], "hello.txt", { type: "text/plain" })); + + expect(sent!.headers.get(UPLOAD_HEADER)).toBe("true"); + expect(received).toBeInstanceOf(File); + expect((received as File).name).toBe("hello.txt"); + expect((received as File).type).toBe("text/plain"); + expect(await (received as File).text()).toBe("hello"); + }); + + it("sends files next to other arguments and inside them", async () => { + const file = new File(["a"], "a.txt"); + const form = new FormData(); + form.set("avatar", new File(["png"], "avatar.png")); + + const [first, meta, received] = await roundTrip( + file, + { tags: ["x"], list: [new Blob(["b"])], map: new Map([["c", new File(["c"], "c.txt")]]) }, + form + ); + + expect(await (first as File).text()).toBe("a"); + const value = meta as { tags: string[]; list: Blob[]; map: Map }; + expect(value.tags).toEqual(["x"]); + expect(await value.list[0]!.text()).toBe("b"); + expect(await value.map.get("c")!.text()).toBe("c"); + expect(await ((received as FormData).get("avatar") as File).text()).toBe("png"); + }); + + it("sends a file used twice once, and keeps it the same value", async () => { + const file = new File(["once"], "once.txt"); + + const [first, second] = await roundTrip(file, { again: file }); + + const form = await sent!.clone().formData(); + expect(form.has(`${FILE_PART}0`)).toBe(true); + expect(form.has(`${FILE_PART}1`)).toBe(false); + expect((second as { again: File }).again).toBe(first); + }); + + it("copies a blob from another frame", async () => { + const source = new Blob(["other"], { type: "text/plain" }); + // Not an instance of this realm's Blob, as one from another frame isn't. + const foreign = { + size: source.size, + type: source.type, + arrayBuffer: () => source.arrayBuffer(), + slice: () => source.slice() + }; + + const [received] = await roundTrip(foreign, 1); + + expect(await (received as Blob).text()).toBe("other"); + }); + + it("keeps calls without files as serialized text", async () => { + const [value, count] = await roundTrip({ a: 1 }, 2); + + expect(sent!.headers.get("x-serialized")).toBe("true"); + expect(sent!.headers.get(UPLOAD_HEADER)).toBeNull(); + expect(value).toEqual({ a: 1 }); + expect(count).toBe(2); + }); + + it("rejects an upload whose part is missing", async () => { + const form = new FormData(); + form.set( + ARGS_PART, + await serializeToJSONString([new File(["x"], "x.txt")], [createUploadSerializer([])]) + ); + + await expect(readUploadArgs(form)).rejects.toThrow(); + }); + + it("rejects an upload without its arguments", async () => { + const form = new FormData(); + form.set(`${FILE_PART}0`, new File(["x"], "x.txt")); + + await expect(readUploadArgs(form)).rejects.toThrow("Malformed server function upload."); + }); + + it("does not read upload indexes from a body that is not multipart", async () => { + const { deserializeFromJSONString } = await import("./serialization"); + const body = await serializeToJSONString( + [new File(["x"], "x.txt")], + [createUploadSerializer([])] + ); + + // A plain body is read without the upload plugin, so the index has no meaning there. + await expect(deserializeFromJSONString(body)).rejects.toThrow(); + }); +}); diff --git a/packages/start/src/runtime/uploads.ts b/packages/start/src/runtime/uploads.ts new file mode 100644 index 000000000..65dc2f5e6 --- /dev/null +++ b/packages/start/src/runtime/uploads.ts @@ -0,0 +1,152 @@ +import { createPlugin, type Plugin, type SerovalNode } from "seroval"; +import { deserializeFromJSONString } from "./serialization"; + +/** + * Files and blobs in the arguments of a server function call go as parts of + * multipart form data instead of inside the serialized arguments. The + * arguments go in the `ARGS_PART` field, and each file in `FILE_PART`, + * where the arguments hold only its index. + */ +export const ARGS_PART = "args"; +export const FILE_PART = "file"; + +/** Marks a call whose serialized arguments come with their files as form parts. */ +export const UPLOAD_HEADER = "x-serialized-form"; + +const UPLOAD_TAG = "solid-start/upload"; + +/* + * Migration note: seroval's binary mode (`binary.serialize` and + * `binary.deserialize`, in development on seroval's `binary-mode` branch) + * could replace this when server function bodies move to it. + * + * Binary mode would be better for the arguments in general: + * - one body and one format for every value, instead of JSON in a form field; + * - `ArrayBuffer` and typed array bytes sent as they are, with no base64 and + * no `maxBase64Length` limit; + * - decoding that validates untrusted input and runs no code, as `fromCrossJSON` + * does today. + * + * For files, the form parts here are still better: + * - `BlobPlugin` and `FilePlugin` read the whole file with `arrayBuffer()` + * before sending it, so a large upload is held in memory on the client. A + * `File` in `FormData` is read by the browser as it sends the request. + * - Sending binary mode output as it is produced needs a streamed request + * body (`duplex: "half"`), which not every browser supports. Otherwise the + * chunks are collected first, which holds the file in memory again. + * - Multipart bodies work in every runtime and with every proxy, and servers + * read them with `request.formData()`. + * + * Neither limits the total size: binary mode leaves that to the transport, + * and `request.formData()` reads the whole body. A limit belongs in the + * server either way. + * + * When migrating, the arguments can move to binary mode while files keep + * going as form parts, with binary mode's plugin section writing the part + * index the way `createUploadSerializer` does. A server should accept both + * bodies for a while, because an open tab can call a newer server. + */ + +type UploadNode = { i: SerovalNode }; + +/** + * True for a file or a blob. One from another frame is an instance of that + * frame's `Blob`, so its shape tells instead. + */ +function isBlob(value: unknown): value is Blob { + if (value instanceof Blob) return true; + const blob = value as Partial | null; + return ( + typeof blob === "object" && + blob !== null && + typeof blob.size === "number" && + typeof blob.type === "string" && + typeof blob.arrayBuffer === "function" && + typeof blob.slice === "function" + ); +} + +function malformed(): Error { + return new Error("Malformed server function upload."); +} + +/** + * Serializes each file or blob in the arguments as its index in `files`, so + * the call can send it as a form part. Seroval calls this wherever it finds + * one: in arrays, objects, maps, sets and form data. + */ +export function createUploadSerializer(files: Blob[]): Plugin { + const parse = (value: Blob) => files.push(value) - 1; + return createPlugin({ + tag: UPLOAD_TAG, + test: isBlob, + parse: { + sync: (value, ctx) => ({ i: ctx.parse(parse(value)) }), + async: async (value, ctx) => ({ i: await ctx.parse(parse(value)) }), + stream: (value, ctx) => ({ i: ctx.parse(parse(value)) }) + }, + serialize: (node, ctx) => ctx.serialize(node.i), + deserialize: () => { + throw malformed(); + } + }); +} + +/** + * Turns each index written by `createUploadSerializer` back into its part of + * `form`. It is only used for a multipart call, so an index in any other body + * fails to deserialize. + */ +export function createUploadDeserializer(form: FormData): Plugin { + return createPlugin({ + tag: UPLOAD_TAG, + test: () => false, + parse: { + sync: () => { + throw malformed(); + }, + async: async () => { + throw malformed(); + }, + stream: () => { + throw malformed(); + } + }, + serialize: () => { + throw malformed(); + }, + deserialize: (node, ctx) => { + const index = ctx.deserialize(node.i); + if (typeof index !== "number" || !Number.isSafeInteger(index) || index < 0) { + throw malformed(); + } + const part = form.get(`${FILE_PART}${index}`); + if (!(part instanceof Blob)) throw malformed(); + return part; + } + }); +} + +/** + * A blob of another frame is copied, because form data only takes blobs of + * its own. + */ +export async function appendUploads(form: FormData, files: Blob[]): Promise { + const blobs = await Promise.all( + files.map(async file => + file instanceof Blob + ? file + : new Blob([await (file as Blob).arrayBuffer()], { type: (file as Blob).type }) + ) + ); + for (const [index, blob] of blobs.entries()) { + form.set(`${FILE_PART}${index}`, blob, (files[index] as Partial).name ?? "blob"); + } +} + +/** Reads the arguments of a call with files: the serialized arguments, with each index replaced by its part. */ +export async function readUploadArgs(form: FormData): Promise { + const args = form.get(ARGS_PART); + if (typeof args !== "string") throw malformed(); + return await deserializeFromJSONString(args, [createUploadDeserializer(form)]); +}