diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index c5aaf70e1fe..fc18b7f09d8 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -210,6 +210,14 @@ export const Env = z KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods + KUBERNETES_RUNNER_SECCOMP_PROFILE_PATH: z + .string() + .trim() + .min(1) + .default("profiles/block-io-uring.json"), + KUBERNETES_RUNNER_SECCOMP_PROFILE_RUNTIMES: z + .enum(["none", "node-24-plus", "all"]) + .default("node-24-plus"), // Pod DNS config — override the cluster default ndots to `KUBERNETES_POD_DNS_NDOTS`. // Default k8s ndots is 5: any name with fewer than 5 dots (e.g. `api.example.com`, 2 dots) is first walked diff --git a/apps/supervisor/src/workloadManager/kubernetes.test.ts b/apps/supervisor/src/workloadManager/kubernetes.test.ts index 3c6419e9c6f..e3023bc5d84 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.test.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.test.ts @@ -1,9 +1,8 @@ import { describe, expect, it } from "vitest"; import { - BLOCK_IO_URING_SECCOMP_PROFILE, nodetypeNodeSelector, runPodTolerations, - withBlockIoUringSeccompProfile, + withRunnerSeccompProfile, withNodeSelector, } from "./kubernetesPodSpec.js"; @@ -100,27 +99,57 @@ describe("withNodeSelector", () => { }); }); -describe("withBlockIoUringSeccompProfile", () => { - it("adds the Localhost io_uring profile for node-24 and above, preserving pod security defaults", () => { +describe("withRunnerSeccompProfile", () => { + const base = { + profilePath: "profiles/example.json", + runtimes: "node-24-plus" as const, + runtime: "node-24", + checkpointsEnabled: true, + }; + + const withProfile = { + ...basePodSpec, + securityContext: { + ...basePodSpec.securityContext, + seccompProfile: { type: "Localhost", localhostProfile: "profiles/example.json" }, + }, + }; + + it("applies the profile to node-24 and above under the default scope", () => { for (const runtime of ["node-24", "node-26", "node-30", "experimental-node-24"]) { - const podSpec = withBlockIoUringSeccompProfile(basePodSpec, runtime); - - expect(podSpec).toMatchObject({ - ...basePodSpec, - securityContext: { - ...basePodSpec.securityContext, - seccompProfile: { - type: "Localhost", - localhostProfile: BLOCK_IO_URING_SECCOMP_PROFILE, - }, - }, - }); + expect(withRunnerSeccompProfile(basePodSpec, { ...base, runtime })).toMatchObject( + withProfile + ); } }); - it("leaves the pod spec unchanged for runtimes that do not create io_uring fds", () => { + it("skips older runtimes under the default scope", () => { for (const runtime of ["node", "node-22", "bun", undefined, null, ""]) { - expect(withBlockIoUringSeccompProfile(basePodSpec, runtime)).toEqual(basePodSpec); + expect(withRunnerSeccompProfile(basePodSpec, { ...base, runtime })).toBe(basePodSpec); + } + }); + + it("applies the profile to every runtime under the all scope", () => { + for (const runtime of ["node", "node-22", "bun", "node-24", undefined]) { + expect( + withRunnerSeccompProfile(basePodSpec, { ...base, runtimes: "all", runtime }) + ).toMatchObject(withProfile); + } + }); + + it("applies nothing under the none scope, whatever the runtime", () => { + for (const runtime of ["node-24", "bun", "node-22"]) { + expect(withRunnerSeccompProfile(basePodSpec, { ...base, runtimes: "none", runtime })).toBe( + basePodSpec + ); + } + }); + + it("applies nothing when checkpoints are disabled", () => { + for (const runtimes of ["none", "node-24-plus", "all"] as const) { + expect( + withRunnerSeccompProfile(basePodSpec, { ...base, runtimes, checkpointsEnabled: false }) + ).toBe(basePodSpec); } }); }); diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index e0bf01a050f..d09c86cf9cc 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -17,7 +17,7 @@ import { getRunnerId } from "../util.js"; import { nodetypeNodeSelector, runPodTolerations, - withBlockIoUringSeccompProfile, + withRunnerSeccompProfile, withNodeSelector, } from "./kubernetesPodSpec.js"; @@ -135,9 +135,12 @@ export class KubernetesWorkloadManager implements WorkloadManager { ); } } - const podSpec = this.opts.checkpointsEnabled - ? withBlockIoUringSeccompProfile(basePodSpec, opts.runtime) - : basePodSpec; + const podSpec = withRunnerSeccompProfile(basePodSpec, { + profilePath: env.KUBERNETES_RUNNER_SECCOMP_PROFILE_PATH, + runtimes: env.KUBERNETES_RUNNER_SECCOMP_PROFILE_RUNTIMES, + runtime: opts.runtime, + checkpointsEnabled: this.opts.checkpointsEnabled, + }); await this.k8s.core.createNamespacedPod({ namespace: this.namespace, diff --git a/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts b/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts index f521369f082..6c6ddfc09b0 100644 --- a/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts +++ b/apps/supervisor/src/workloadManager/kubernetesPodSpec.ts @@ -1,11 +1,5 @@ import type { k8s } from "../clients/kubernetes.js"; -/** - * Relative path (kubelet seccomp root) of the profile blocking only io_uring - * syscalls. Must match the profile deployed to worker nodes. - */ -export const BLOCK_IO_URING_SECCOMP_PROFILE = "profiles/block-io-uring.json"; - /** * An empty label is the documented off-switch, leaving the pod unpinned. The Helm * chart ships an empty value, so don't collapse this into a fallback default - @@ -60,27 +54,44 @@ export function withNodeSelector( }; } +export type RunnerSeccompProfileOptions = { + profilePath: string; + runtimes: "none" | "node-24-plus" | "all"; + runtime: string | null | undefined; + checkpointsEnabled: boolean | undefined; +}; + /** - * Node >= 24 always creates io_uring fds, which can't be checkpointed. Blocking - * io_uring_setup makes libuv fall back to epoll. Other runtimes don't need this, - * so the profile is only applied for node-24+. Tolerates an "experimental-" prefix. + * Applies the runner seccomp profile, which is a node-local file installed outside + * this repo - pointing a pod at a profile its node doesn't have fails pod creation, + * so every condition for skipping it lives here. + * + * "node-24-plus" matches the original rollout: node >= 24 always creates io_uring + * fds, which can't be checkpointed, and blocking io_uring_setup makes libuv fall + * back to epoll. Tolerates an "experimental-" prefix. "bun" matches only under "all". */ -export function withBlockIoUringSeccompProfile( +export function withRunnerSeccompProfile( podSpec: Omit, - runtime: string | null | undefined + options: RunnerSeccompProfileOptions ): Omit { - const match = runtime ? /^(?:experimental-)?node-(\d+)$/.exec(runtime) : null; - if (!match || Number(match[1]) < 24) { + if (!options.checkpointsEnabled || options.runtimes === "none") { return podSpec; } + if (options.runtimes === "node-24-plus") { + const match = options.runtime ? /^(?:experimental-)?node-(\d+)$/.exec(options.runtime) : null; + if (!match || Number(match[1]) < 24) { + return podSpec; + } + } + return { ...podSpec, securityContext: { ...podSpec.securityContext, seccompProfile: { type: "Localhost", - localhostProfile: BLOCK_IO_URING_SECCOMP_PROFILE, + localhostProfile: options.profilePath, }, }, };