Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apps/supervisor/src/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,14 @@ export const Env = z

KUBERNETES_MEMORY_OVERHEAD_GB: z.coerce.number().min(0).optional(), // Optional memory overhead to add to the limit in GB
KUBERNETES_SCHEDULER_NAME: z.string().optional(), // Custom scheduler name for pods
KUBERNETES_RUNNER_SECCOMP_PROFILE_PATH: z
.string()
.trim()
.min(1)
.default("profiles/block-io-uring.json"),
Comment thread
nicktrn marked this conversation as resolved.
Comment thread
coderabbitai[bot] marked this conversation as resolved.
KUBERNETES_RUNNER_SECCOMP_PROFILE_RUNTIMES: z
.enum(["none", "node-24-plus", "all"])
.default("node-24-plus"),

// Pod DNS config — override the cluster default ndots to `KUBERNETES_POD_DNS_NDOTS`.
// Default k8s ndots is 5: any name with fewer than 5 dots (e.g. `api.example.com`, 2 dots) is first walked
Expand Down
65 changes: 47 additions & 18 deletions apps/supervisor/src/workloadManager/kubernetes.test.ts
Original file line number Diff line number Diff line change
@@ -1,9 +1,8 @@
import { describe, expect, it } from "vitest";
import {
BLOCK_IO_URING_SECCOMP_PROFILE,
nodetypeNodeSelector,
runPodTolerations,
withBlockIoUringSeccompProfile,
withRunnerSeccompProfile,
withNodeSelector,
} from "./kubernetesPodSpec.js";

Expand Down Expand Up @@ -100,27 +99,57 @@ describe("withNodeSelector", () => {
});
});

describe("withBlockIoUringSeccompProfile", () => {
it("adds the Localhost io_uring profile for node-24 and above, preserving pod security defaults", () => {
describe("withRunnerSeccompProfile", () => {
const base = {
profilePath: "profiles/example.json",
runtimes: "node-24-plus" as const,
runtime: "node-24",
checkpointsEnabled: true,
};

const withProfile = {
...basePodSpec,
securityContext: {
...basePodSpec.securityContext,
seccompProfile: { type: "Localhost", localhostProfile: "profiles/example.json" },
},
};

it("applies the profile to node-24 and above under the default scope", () => {
for (const runtime of ["node-24", "node-26", "node-30", "experimental-node-24"]) {
const podSpec = withBlockIoUringSeccompProfile(basePodSpec, runtime);

expect(podSpec).toMatchObject({
...basePodSpec,
securityContext: {
...basePodSpec.securityContext,
seccompProfile: {
type: "Localhost",
localhostProfile: BLOCK_IO_URING_SECCOMP_PROFILE,
},
},
});
expect(withRunnerSeccompProfile(basePodSpec, { ...base, runtime })).toMatchObject(
withProfile
);
}
});

it("leaves the pod spec unchanged for runtimes that do not create io_uring fds", () => {
it("skips older runtimes under the default scope", () => {
for (const runtime of ["node", "node-22", "bun", undefined, null, ""]) {
expect(withBlockIoUringSeccompProfile(basePodSpec, runtime)).toEqual(basePodSpec);
expect(withRunnerSeccompProfile(basePodSpec, { ...base, runtime })).toBe(basePodSpec);
}
});

it("applies the profile to every runtime under the all scope", () => {
for (const runtime of ["node", "node-22", "bun", "node-24", undefined]) {
expect(
withRunnerSeccompProfile(basePodSpec, { ...base, runtimes: "all", runtime })
).toMatchObject(withProfile);
}
});

it("applies nothing under the none scope, whatever the runtime", () => {
for (const runtime of ["node-24", "bun", "node-22"]) {
expect(withRunnerSeccompProfile(basePodSpec, { ...base, runtimes: "none", runtime })).toBe(
basePodSpec
);
}
});

it("applies nothing when checkpoints are disabled", () => {
for (const runtimes of ["none", "node-24-plus", "all"] as const) {
expect(
withRunnerSeccompProfile(basePodSpec, { ...base, runtimes, checkpointsEnabled: false })
).toBe(basePodSpec);
}
});
});
11 changes: 7 additions & 4 deletions apps/supervisor/src/workloadManager/kubernetes.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ import { getRunnerId } from "../util.js";
import {
nodetypeNodeSelector,
runPodTolerations,
withBlockIoUringSeccompProfile,
withRunnerSeccompProfile,
withNodeSelector,
} from "./kubernetesPodSpec.js";

Expand Down Expand Up @@ -135,9 +135,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
);
}
}
const podSpec = this.opts.checkpointsEnabled
? withBlockIoUringSeccompProfile(basePodSpec, opts.runtime)
: basePodSpec;
const podSpec = withRunnerSeccompProfile(basePodSpec, {
profilePath: env.KUBERNETES_RUNNER_SECCOMP_PROFILE_PATH,
runtimes: env.KUBERNETES_RUNNER_SECCOMP_PROFILE_RUNTIMES,
runtime: opts.runtime,
checkpointsEnabled: this.opts.checkpointsEnabled,
});
Comment thread
nicktrn marked this conversation as resolved.

await this.k8s.core.createNamespacedPod({
namespace: this.namespace,
Expand Down
39 changes: 25 additions & 14 deletions apps/supervisor/src/workloadManager/kubernetesPodSpec.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,5 @@
import type { k8s } from "../clients/kubernetes.js";

/**
* Relative path (kubelet seccomp root) of the profile blocking only io_uring
* syscalls. Must match the profile deployed to worker nodes.
*/
export const BLOCK_IO_URING_SECCOMP_PROFILE = "profiles/block-io-uring.json";

/**
* An empty label is the documented off-switch, leaving the pod unpinned. The Helm
* chart ships an empty value, so don't collapse this into a fallback default -
Expand Down Expand Up @@ -60,27 +54,44 @@ export function withNodeSelector(
};
}

export type RunnerSeccompProfileOptions = {
profilePath: string;
runtimes: "none" | "node-24-plus" | "all";
runtime: string | null | undefined;
checkpointsEnabled: boolean | undefined;
};

/**
* Node >= 24 always creates io_uring fds, which can't be checkpointed. Blocking
* io_uring_setup makes libuv fall back to epoll. Other runtimes don't need this,
* so the profile is only applied for node-24+. Tolerates an "experimental-" prefix.
* Applies the runner seccomp profile, which is a node-local file installed outside
* this repo - pointing a pod at a profile its node doesn't have fails pod creation,
* so every condition for skipping it lives here.
*
* "node-24-plus" matches the original rollout: node >= 24 always creates io_uring
* fds, which can't be checkpointed, and blocking io_uring_setup makes libuv fall
* back to epoll. Tolerates an "experimental-" prefix. "bun" matches only under "all".
*/
export function withBlockIoUringSeccompProfile(
export function withRunnerSeccompProfile(
podSpec: Omit<k8s.V1PodSpec, "containers">,
runtime: string | null | undefined
options: RunnerSeccompProfileOptions
): Omit<k8s.V1PodSpec, "containers"> {
const match = runtime ? /^(?:experimental-)?node-(\d+)$/.exec(runtime) : null;
if (!match || Number(match[1]) < 24) {
if (!options.checkpointsEnabled || options.runtimes === "none") {
return podSpec;
}

if (options.runtimes === "node-24-plus") {
const match = options.runtime ? /^(?:experimental-)?node-(\d+)$/.exec(options.runtime) : null;
if (!match || Number(match[1]) < 24) {
return podSpec;
}
}
Comment thread
nicktrn marked this conversation as resolved.

return {
...podSpec,
securityContext: {
...podSpec.securityContext,
seccompProfile: {
type: "Localhost",
localhostProfile: BLOCK_IO_URING_SECCOMP_PROFILE,
localhostProfile: options.profilePath,
},
},
};
Expand Down