Skip to content

Commit 00e3c15

Browse files
d-csclaude
andauthored
feat(webapp): RUN_OPS_SHARDS config, topology and N-way store wiring (#4764)
Part of the RunOps N-way sharding work. This lets the webapp hold N run-ops stores, configured by a single `RUN_OPS_SHARDS` JSON descriptor, and routes to them through the existing keyed router. **Inert with `RUN_OPS_SHARDS` unset** — the topology, the wiring and `ROUTING_ENABLED` are byte-identical to today. ## What's here - **`RUN_OPS_SHARDS`** — a zod-validated JSON array of shard descriptors (`key`, `region`, `url`, `replicaUrl`, `directUrl`, `replication`, `knobs`, `aliasOf`), validated at boot in the `parseMachinePresetCsv` style. Unset or `[]` → no shards. - **One run-ops client factory** — `buildRunOpsWriterClient`/`buildRunOpsReplicaClient` collapse into one `buildRunOpsClient` parameterized by role and resolved pool knobs. The control-plane builders (`buildWriterClient`/`buildReplicaClient`) are a separate path and stay untouched; every resolved value matches the former builders. - **Shard loop in `selectRunOpsTopology`** — one client pair per descriptor; an `aliasOf: "new"` descriptor reuses the new store's clients by reference and opens no pool. - **N-way `buildRunStore`** — builds N dedicated stores + the keyed router via a new `RoutingRunStore.fromShards`, keeping the two-store compat router when no shards are configured. - **`UnknownShardKey`** — raised when an id resolves to an unconfigured key; never falls back to another store. `fromShards` injects `resolveShard` so a gen-2 id routes to its own shard. - **Per-shard transaction resilience** — each shard gets its own retry budget. - **Mint bound** — `computeMintShard` intersects the active mint list with the configured descriptor keys, so a key with no descriptor is never minted into. - **Boot table** — logs `key`, address fingerprint (host:port/db, no credentials), and role, only when shards are configured. ## Ordering constraint Do **not** configure a `RUN_OPS_SHARDS` descriptor in any environment until the routing-semantics change (TRI-13427) lands — three fan-out sites still truncate at N>2. Merging this PR alone is safe (inert with the var unset); configuring a descriptor is what must wait. ## Testing - Run-store corpus: green with zero test-file diffs (the bit-identical proof for the compat router). - `runOpsDbTopology.test.ts` 17/17, `runStore.server.test.ts` 4/4, `runOpsMigration` family 149/149. - New unit suites: descriptor validation, pool-knob value tables, `fromShards` routing + `UnknownShardKey`, boot-table formatter, mint bound. - typecheck (webapp + run-store), knip, lint, format: pass. ## Changelog Internal run-ops sharding infrastructure. No changeset or `.server-changes`: the change is inert with `RUN_OPS_SHARDS` unset and has no user-visible behaviour. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent ba57c1f commit 00e3c15

18 files changed

Lines changed: 874 additions & 159 deletions

apps/webapp/app/db.server.ts

Lines changed: 183 additions & 155 deletions
Large diffs are not rendered by default.

apps/webapp/app/env.server.ts

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import { z } from "zod";
22
import { MachinePresetName } from "@trigger.dev/core/v3";
33
import { BoolEnv } from "./utils/boolEnv";
44
import { isValidDatabaseUrl } from "./utils/db";
5+
import { parseRunOpsShards, validateShardListAgainstNewUrl } from "~/v3/runOpsShards.server";
56
import { isValidRegex } from "./utils/regex";
67
import { isValidDuration } from "./services/realtime/duration.server";
78

@@ -310,6 +311,8 @@ const EnvironmentSchema = z
310311
RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"),
311312
RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER: z.string().default("0"),
312313
RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER: z.string().default("0"),
314+
// Gen-2 shard descriptors as a JSON array. Unset/"" -> [] (today). See runOpsShards.server.ts.
315+
RUN_OPS_SHARDS: z.string().optional().transform(parseRunOpsShards),
313316
// Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES).
314317
CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(),
315318
CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(),
@@ -2491,6 +2494,14 @@ const EnvironmentSchema = z
24912494
});
24922495
}
24932496
}
2497+
if (!validateShardListAgainstNewUrl(env.RUN_OPS_SHARDS, env.RUN_OPS_DATABASE_URL)) {
2498+
ctx.addIssue({
2499+
code: z.ZodIssueCode.custom,
2500+
path: ["RUN_OPS_SHARDS"],
2501+
message:
2502+
"RUN_OPS_SHARDS is non-empty but RUN_OPS_DATABASE_URL is unset; a shard requires the gen-1 new store",
2503+
});
2504+
}
24942505
});
24952506

24962507
export type Environment = z.infer<typeof EnvironmentSchema>;

apps/webapp/app/v3/runOpsMigration/mintShardAssignment.test.ts

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -473,3 +473,40 @@ describe("computeMintShard — the global override wins the complete cutover", (
473473
);
474474
});
475475
});
476+
477+
describe("routableKeys bound (the shard descriptor keys this deployment can route)", () => {
478+
it("drops an active key that is not routable, so the hash never returns it", () => {
479+
// "z" is in the active list but not configured as a descriptor -> only "a" is selectable.
480+
const ids = envIds(200);
481+
for (const id of ids) {
482+
const shard = computeMintShard({ id }, deps({ set: ["a", "z"] }, { routableKeys: ["a"] }));
483+
expect(shard).toBe("a");
484+
}
485+
});
486+
487+
it("returns new when the active list holds only non-routable keys (fail-safe to gen-1)", () => {
488+
expect(computeMintShard({ id: "env_1" }, deps({ set: ["z"] }, { routableKeys: ["a"] }))).toBe(
489+
"new"
490+
);
491+
});
492+
493+
it("rejects a per-org pin to a non-routable key and falls through to the hash", () => {
494+
const shard = computeMintShard(
495+
{ id: "env_1" },
496+
deps({ set: ["a", "z"] }, { ...orgFlags({ runOpsMintShard: "z" }), routableKeys: ["a"] })
497+
);
498+
expect(shard).toBe("a");
499+
});
500+
501+
it("with no routableKeys given, behaviour is unchanged", () => {
502+
const ids = envIds(200);
503+
for (const id of ids) {
504+
const withBound = computeMintShard(
505+
{ id },
506+
deps({ set: ["a", "b"] }, { routableKeys: ["a", "b"] })
507+
);
508+
const without = computeMintShard({ id }, deps({ set: ["a", "b"] }));
509+
expect(withBound).toBe(without);
510+
}
511+
});
512+
});

apps/webapp/app/v3/runOpsMigration/mintShardAssignment.ts

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ export type MintShardDeps = {
1919
nowMs: number;
2020
graceMs: number;
2121
orgFeatureFlags: unknown;
22+
// The shard keys this deployment can actually route (the RUN_OPS_SHARDS descriptor keys). The
23+
// active set is bounded to these, so a stored key with no descriptor is never minted into.
24+
// Undefined means "no bound" (today's behaviour).
25+
routableKeys?: readonly string[];
2226
onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void;
2327
onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void;
2428
};
@@ -94,7 +98,17 @@ function hrwSelect(environmentId: string, activeSet: string[]): string {
9498
// would leak the drain the active list performs, and throwing would fail customer triggers
9599
// whenever a pinned shard drains.
96100
export function computeMintShard(environment: { id: string }, deps: MintShardDeps): ShardKey {
97-
const activeSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs);
101+
const rawActiveSet = effectiveMintShardSet(deps.resolution, deps.nowMs, deps.graceMs);
102+
// Empty check BEFORE the bound, so an unconfigured deployment returns "new" exactly as today.
103+
if (rawActiveSet.length === 0) {
104+
return "new";
105+
}
106+
107+
// Bound the active set to the keys this deployment can route. A stored key with no descriptor is
108+
// dropped, never minted into. If nothing survives, fall back to gen-1 (fail-safe, never a throw).
109+
const activeSet = deps.routableKeys
110+
? rawActiveSet.filter((key) => deps.routableKeys!.includes(key))
111+
: rawActiveSet;
98112
if (activeSet.length === 0) {
99113
return "new";
100114
}
@@ -148,6 +162,7 @@ export type ResolveMintShardDeps = {
148162
ttlMs: number;
149163
graceMs: number;
150164
orgFeatureFlags: unknown;
165+
routableKeys?: readonly string[];
151166
onPinRejected?: (info: { environmentId: string; pin: string; activeSet: string[] }) => void;
152167
onOverrideRejected?: (info: { override: string; activeSet: string[] }) => void;
153168
onReadFailed?: (error: unknown) => void;
@@ -200,6 +215,7 @@ export async function resolveMintShardWith(
200215
nowMs: deps.nowMs,
201216
graceMs: deps.graceMs,
202217
orgFeatureFlags: deps.orgFeatureFlags,
218+
routableKeys: deps.routableKeys,
203219
onPinRejected: deps.onPinRejected,
204220
onOverrideRejected: deps.onOverrideRejected,
205221
});

apps/webapp/app/v3/runOpsMigration/runOpsMintShard.server.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ export async function resolveMintShard(environment: {
8484
ttlMs: env.RUN_OPS_MINT_FLAG_CACHE_TTL_MS,
8585
graceMs: env.RUN_OPS_MINT_FLIP_GRACE_MS,
8686
orgFeatureFlags: environment.orgFeatureFlags,
87+
// Bound the active list to the shards this deployment can actually route.
88+
routableKeys: env.RUN_OPS_SHARDS.map((shard) => shard.key),
8789
onPinRejected: reportPinRejected,
8890
onOverrideRejected: reportOverrideRejected,
8991
onReadFailed: (error) =>
Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
import { env } from "~/env.server";
2+
import type { RunOpsShardKnobs } from "~/v3/runOpsShards.server";
3+
4+
// Pool configuration for one run-ops store (writer + replica). Kept separate from db.server (which
5+
// ~156 tests mock wholesale) so a new export breaks no mock.
6+
export type ResolvedPoolKnobs = {
7+
writerPoolTimeout: number;
8+
writerConnectionTimeout: number;
9+
writerDriverAdapter: boolean;
10+
connectionLimit: number;
11+
replicaConnectionLimit: number;
12+
replicaPoolTimeout: number;
13+
replicaConnectionTimeout: number;
14+
replicaDriverAdapter: boolean;
15+
};
16+
17+
type Role = "new" | "legacy";
18+
19+
// PURE: overlay a gen-2 shard's descriptor knobs on a role's resolved defaults. This holds the only
20+
// logic (per-field override), so a test drives it with literal defaults and literal overrides —
21+
// no env import, no circular assertion against the same env expression the impl reads.
22+
export function applyPoolKnobOverrides(
23+
defaults: ResolvedPoolKnobs,
24+
k?: RunOpsShardKnobs
25+
): ResolvedPoolKnobs {
26+
return {
27+
writerPoolTimeout: k?.writerPoolTimeout ?? defaults.writerPoolTimeout,
28+
writerConnectionTimeout: k?.writerConnectionTimeout ?? defaults.writerConnectionTimeout,
29+
writerDriverAdapter: k?.writerDriverAdapter ?? defaults.writerDriverAdapter,
30+
connectionLimit: k?.connectionLimit ?? defaults.connectionLimit,
31+
replicaConnectionLimit: k?.replicaConnectionLimit ?? defaults.replicaConnectionLimit,
32+
replicaPoolTimeout: k?.replicaPoolTimeout ?? defaults.replicaPoolTimeout,
33+
replicaConnectionTimeout: k?.replicaConnectionTimeout ?? defaults.replicaConnectionTimeout,
34+
replicaDriverAdapter: k?.replicaDriverAdapter ?? defaults.replicaDriverAdapter,
35+
};
36+
}
37+
38+
// The env-derived defaults for a role, reproducing today's run-ops builder expressions exactly. A
39+
// flat mapping (no logic), verified by inspection against the former builders. Transaction
40+
// resilience is a SEPARATE mechanism (resolveTransactionResilience) and is not here.
41+
function poolKnobDefaults(role: Role): ResolvedPoolKnobs {
42+
if (role === "legacy") {
43+
return {
44+
writerPoolTimeout:
45+
env.RUN_OPS_LEGACY_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
46+
writerConnectionTimeout:
47+
env.RUN_OPS_LEGACY_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT,
48+
writerDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_WRITER_DRIVER_ADAPTER === "1",
49+
connectionLimit: env.DATABASE_CONNECTION_LIMIT,
50+
replicaConnectionLimit: env.DATABASE_CONNECTION_LIMIT,
51+
replicaPoolTimeout:
52+
env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
53+
replicaConnectionTimeout:
54+
env.RUN_OPS_LEGACY_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ??
55+
env.DATABASE_CONNECTION_TIMEOUT,
56+
replicaDriverAdapter: env.RUN_OPS_LEGACY_DATABASE_REPLICA_DRIVER_ADAPTER === "1",
57+
};
58+
}
59+
60+
return {
61+
writerPoolTimeout: env.RUN_OPS_DATABASE_WRITER_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
62+
writerConnectionTimeout:
63+
env.RUN_OPS_DATABASE_WRITER_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT,
64+
writerDriverAdapter: env.RUN_OPS_DATABASE_WRITER_DRIVER_ADAPTER === "1",
65+
connectionLimit: env.DATABASE_CONNECTION_LIMIT,
66+
replicaConnectionLimit:
67+
env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_LIMIT ?? env.DATABASE_CONNECTION_LIMIT,
68+
replicaPoolTimeout: env.RUN_OPS_DATABASE_READ_REPLICA_POOL_TIMEOUT ?? env.DATABASE_POOL_TIMEOUT,
69+
replicaConnectionTimeout:
70+
env.RUN_OPS_DATABASE_READ_REPLICA_CONNECTION_TIMEOUT ?? env.DATABASE_CONNECTION_TIMEOUT,
71+
replicaDriverAdapter: env.RUN_OPS_DATABASE_REPLICA_DRIVER_ADAPTER === "1",
72+
};
73+
}
74+
75+
export function resolveRunOpsPoolKnobs(
76+
role: Role,
77+
descriptorKnobs?: RunOpsShardKnobs
78+
): ResolvedPoolKnobs {
79+
return applyPoolKnobOverrides(poolKnobDefaults(role), descriptorKnobs);
80+
}
Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Pure boot-table helpers. Dependency-free (no db.server, no env) so a test of these two string
2+
// functions never constructs a Prisma client. db.server imports them for the boot log.
3+
4+
// A host:port/db address, with NO username and NO query params — never a secret, and deliberately
5+
// NOT an identity claim (two DSNs can share an address yet be different databases; that proof is the
6+
// distinctness sentinel's, not this line's). Same tuple sameDatabaseTarget compares, kept in step.
7+
export function runOpsAddressFingerprint(url: string): string {
8+
try {
9+
const u = new URL(url);
10+
return `${u.hostname}:${u.port || "5432"}${u.pathname}`;
11+
} catch {
12+
return "unparseable";
13+
}
14+
}
15+
16+
export type RunOpsShardTableRow = { key: string; fingerprint: string; role: string };
17+
18+
// The resolved shard table for the boot log: one row per descriptor. An alias reports its role and
19+
// carries no address (it shares the new store's pool).
20+
export function buildRunOpsShardTable(
21+
descriptors: Array<{ key: string; url?: string; aliasOf?: "new" }>
22+
): RunOpsShardTableRow[] {
23+
return descriptors.map((d) =>
24+
d.aliasOf
25+
? { key: d.key, fingerprint: "alias(new)", role: "alias(new)" }
26+
: { key: d.key, fingerprint: runOpsAddressFingerprint(d.url ?? ""), role: "shard" }
27+
);
28+
}
Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
import { z } from "zod";
2+
import { isValidShardChar } from "@trigger.dev/core/v3/isomorphic";
3+
import { isValidDatabaseUrl } from "~/utils/db";
4+
5+
const KnobsSchema = z
6+
.object({
7+
writerPoolTimeout: z.number().int().optional(),
8+
writerConnectionTimeout: z.number().int().optional(),
9+
writerDriverAdapter: z.boolean().optional(),
10+
connectionLimit: z.number().int().optional(),
11+
replicaConnectionLimit: z.number().int().optional(),
12+
replicaPoolTimeout: z.number().int().optional(),
13+
replicaConnectionTimeout: z.number().int().optional(),
14+
replicaDriverAdapter: z.boolean().optional(),
15+
transactionMaxWaitMs: z.number().int().optional(),
16+
transactionStartRetryEnabled: z.boolean().optional(),
17+
transactionStartRetryMaxAttempts: z.number().int().optional(),
18+
transactionStartRetryBackoffMinMs: z.number().int().optional(),
19+
transactionStartRetryBackoffMaxMs: z.number().int().optional(),
20+
transactionStartRetryBudgetPerSec: z.number().int().optional(),
21+
transactionStartRetryBudgetBurst: z.number().int().optional(),
22+
})
23+
.strict();
24+
export type RunOpsShardKnobs = z.infer<typeof KnobsSchema>;
25+
26+
const ReplicationSchema = z.object({
27+
slotName: z.string().min(1),
28+
publicationName: z.string().min(1),
29+
originGeneration: z.number().int().min(2).max(255),
30+
});
31+
32+
const DescriptorSchema = z
33+
.object({
34+
key: z.string().refine(isValidShardChar, "shard key must be a single [a-z0-9] char"),
35+
region: z.string().min(1),
36+
url: z.string().refine(isValidDatabaseUrl, "url is invalid").optional(),
37+
replicaUrl: z.string().refine(isValidDatabaseUrl, "replicaUrl is invalid").optional(),
38+
directUrl: z.string().refine(isValidDatabaseUrl, "directUrl is invalid").optional(),
39+
replication: ReplicationSchema.optional(),
40+
knobs: KnobsSchema.optional(),
41+
aliasOf: z.literal("new").optional(),
42+
})
43+
.strict()
44+
.superRefine((d, ctx) => {
45+
const hasUrl = d.url !== undefined;
46+
const hasAlias = d.aliasOf !== undefined;
47+
if (hasUrl === hasAlias) {
48+
ctx.addIssue({
49+
code: z.ZodIssueCode.custom,
50+
message: "exactly one of url or aliasOf is required",
51+
});
52+
}
53+
if (!hasAlias && d.replication === undefined) {
54+
ctx.addIssue({
55+
code: z.ZodIssueCode.custom,
56+
message: "replication is required unless aliasOf is set",
57+
});
58+
}
59+
});
60+
61+
export type RunOpsShardDescriptor = z.infer<typeof DescriptorSchema>;
62+
63+
// Boot-validated transform, in the style of parseMachinePresetCsv. Undefined and "" both mean the
64+
// off state and resolve to []. The undefined guard is load-bearing: an unguarded JSON.parse would
65+
// kill every single-DB boot, which never sets this variable.
66+
export function parseRunOpsShards(
67+
raw: string | undefined,
68+
ctx: z.RefinementCtx
69+
): RunOpsShardDescriptor[] {
70+
if (raw === undefined || raw.trim() === "") return [];
71+
72+
let parsed: unknown;
73+
try {
74+
parsed = JSON.parse(raw);
75+
} catch {
76+
ctx.addIssue({ code: z.ZodIssueCode.custom, message: "RUN_OPS_SHARDS is not valid JSON" });
77+
return z.NEVER;
78+
}
79+
80+
const result = z.array(DescriptorSchema).safeParse(parsed);
81+
if (!result.success) {
82+
for (const issue of result.error.issues) {
83+
ctx.addIssue({
84+
code: z.ZodIssueCode.custom,
85+
message: `RUN_OPS_SHARDS[${issue.path.join(".")}]: ${issue.message}`,
86+
});
87+
}
88+
return z.NEVER;
89+
}
90+
91+
const keys = new Set<string>();
92+
const gens = new Set<number>();
93+
for (const d of result.data) {
94+
if (keys.has(d.key)) {
95+
ctx.addIssue({
96+
code: z.ZodIssueCode.custom,
97+
message: `RUN_OPS_SHARDS: duplicate key ${d.key}`,
98+
});
99+
return z.NEVER;
100+
}
101+
keys.add(d.key);
102+
if (d.replication) {
103+
if (gens.has(d.replication.originGeneration)) {
104+
ctx.addIssue({
105+
code: z.ZodIssueCode.custom,
106+
message: `RUN_OPS_SHARDS: duplicate originGeneration ${d.replication.originGeneration}`,
107+
});
108+
return z.NEVER;
109+
}
110+
gens.add(d.replication.originGeneration);
111+
}
112+
}
113+
114+
return result.data;
115+
}
116+
117+
// A non-empty shard list requires the gen-1 new store, because gen-1 v1 ids resolve to "new"
118+
// forever (append-only). Pure so the boot refinement and its test share one rule.
119+
export function validateShardListAgainstNewUrl(
120+
shards: RunOpsShardDescriptor[],
121+
newUrl: string | undefined
122+
): boolean {
123+
return shards.length === 0 || !!newUrl;
124+
}

0 commit comments

Comments
 (0)