Skip to content

Commit 7a609bf

Browse files
committed
perf(webapp): compute delivery createdAt bounds from the two extreme ids
The delivery-id bounds calc decoded every id in the set to find the createdAt span. Delivery id bodies are base32hex(big-endian timestamp then random bytes), and base32hex is order-preserving, so lexical order equals chronological order: the earliest and latest timestamps sit at the lexical extremes. Track the min and max id in a single pass and decode only those two. The list-hydration query shares the same single-pass min/max helper for its already-decoded page timestamps, dropping the last Math.min(...spread) in the delivery repository (the spread overflows the call stack on large inputs).
1 parent 7fdf3c2 commit 7a609bf

3 files changed

Lines changed: 78 additions & 41 deletions

File tree

apps/webapp/app/services/webhookDeliveriesRepository/clickhouseWebhookDeliveriesRepository.server.ts

Lines changed: 7 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { type ClickhouseQueryBuilder } from "@internal/clickhouse";
22
import { WebhookDeliveryId } from "@trigger.dev/core/v3/isomorphic";
33
import { boundedIn } from "@trigger.dev/database";
4-
import { deliveryIdsCreatedAtBounds } from "./deliveryIdBounds";
4+
import { createdAtMsBounds, deliveryIdsCreatedAtBounds } from "./deliveryIdBounds";
55
import { decodeRunsCursor, encodeRunsCursor } from "../runsRepository/runsCursor.server";
66
import {
77
type CountDeliveriesByEndpointOptions,
@@ -195,18 +195,13 @@ export class ClickHouseWebhookDeliveriesRepository implements IWebhookDeliveries
195195
// `id IN (...)` query without a createdAt predicate scans every child
196196
// partition, so derive a [min, max] range from the CH page and pass it
197197
// through. This is the one place webhook hydration diverges from runs.
198-
const createdAtMsValues = pageRows.map((row) => row.createdAt);
199-
const minCreatedAtMs = Math.min(...createdAtMsValues);
200-
const maxCreatedAtMs = Math.max(...createdAtMsValues);
198+
const bounds = createdAtMsBounds(pageRows.map((row) => row.createdAt));
201199

202200
// CH gives the ordered id list; Postgres hydrates the full lean rows by PK id.
203201
const deliveries = await this.options.prisma.webhookDelivery.findMany({
204202
where: {
205203
id: { in: boundedIn(deliveryIds) },
206-
createdAt: {
207-
gte: new Date(minCreatedAtMs),
208-
lte: new Date(maxCreatedAtMs),
209-
},
204+
...(bounds ? { createdAt: bounds } : {}),
210205
},
211206
select: DELIVERY_LIST_SELECT,
212207
});
@@ -285,8 +280,7 @@ export class ClickHouseWebhookDeliveriesRepository implements IWebhookDeliveries
285280
* `WebhookDelivery` is RANGE-partitioned on `createdAt`, so a bare `id` predicate can't prune and
286281
* probes every partition. The delivery id is time-encoded (see `WebhookDeliveryId`) with the same
287282
* timestamp the engine stores as `createdAt`, so we recover it from the id and add it as an exact
288-
* predicate to prune to the row's partition. A legacy id that doesn't decode falls back to the
289-
* unpruned lookup.
283+
* predicate to prune to the row's partition.
290284
*/
291285
async getDelivery(options: GetWebhookDeliveryOptions): Promise<DetailedWebhookDelivery | null> {
292286
const id = WebhookDeliveryId.toId(options.friendlyId);
@@ -306,10 +300,9 @@ export class ClickHouseWebhookDeliveriesRepository implements IWebhookDeliveries
306300
* Hydrate a known set of deliveries by friendlyId. Pure Postgres: the caller (the live poll)
307301
* already has the ids, so there is nothing for ClickHouse to filter or order.
308302
*
309-
* The ids are time-encoded (see `WebhookDeliveryId`), so when every id decodes we bound the query
310-
* to the span of their mint timestamps, which equal the rows' `createdAt`. That prunes the
311-
* RANGE-partitioned table to the visible page's few days instead of probing all of retention on
312-
* every poll. A mix that includes a legacy id falls back to the unbounded lookup.
303+
* The ids are time-encoded (see `WebhookDeliveryId`), so we bound the query to the span of their
304+
* mint timestamps, which equal the rows' `createdAt`. That prunes the RANGE-partitioned table to
305+
* the visible page's few days instead of probing all of retention on every poll.
313306
*/
314307
async getDeliveriesByFriendlyIds(
315308
options: GetDeliveriesByFriendlyIdsOptions
Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,32 +1,55 @@
11
import { WebhookDeliveryId } from "@trigger.dev/core/v3/isomorphic";
22

33
/**
4-
* Compute the `createdAt` span covering a set of webhook delivery friendlyIds, for partition-pruning a
5-
* lookup by id on the RANGE-partitioned `WebhookDelivery` table.
4+
* Single-pass min/max over a set of `createdAt` timestamps (unix ms), returned as a Prisma
5+
* `{ gte, lte }` range for partition-pruning the RANGE-partitioned `WebhookDelivery` table.
66
*
7-
* Each v1 id is time-encoded (see `WebhookDeliveryId`) with the same timestamp the engine stores as the
8-
* row's `createdAt`, so the returned `[gte, lte]` covers every row in the set exactly. Returns
9-
* `undefined` when the set is empty or contains any legacy (non-time-encoded) id: in that case the
10-
* caller must not add a `createdAt` predicate, since a bound derived from only the decodable ids would
11-
* wrongly exclude the legacy rows.
12-
*
13-
* Single pass, no intermediate arrays and no `Math.min(...spread)` (which is O(n) to build the argument
14-
* list and can overflow the call stack for large inputs).
7+
* Avoids `Math.min(...spread)` / `Math.max(...spread)`: the spread builds an O(n) argument list and
8+
* throws "Maximum call stack size exceeded" once the array is large (~1e5+ elements). Returns
9+
* `undefined` for an empty set, so the caller adds no `createdAt` predicate.
1510
*/
16-
export function deliveryIdsCreatedAtBounds(
17-
friendlyIds: string[]
18-
): { gte: Date; lte: Date } | undefined {
11+
export function createdAtMsBounds(msValues: number[]): { gte: Date; lte: Date } | undefined {
1912
let min = Number.POSITIVE_INFINITY;
2013
let max = Number.NEGATIVE_INFINITY;
2114

22-
for (const friendlyId of friendlyIds) {
23-
const timestamp = WebhookDeliveryId.parseTimestamp(friendlyId);
24-
if (!timestamp) return undefined;
25-
const ms = timestamp.getTime();
15+
for (const ms of msValues) {
2616
if (ms < min) min = ms;
2717
if (ms > max) max = ms;
2818
}
2919

3020
if (min === Number.POSITIVE_INFINITY) return undefined;
3121
return { gte: new Date(min), lte: new Date(max) };
3222
}
23+
24+
/**
25+
* Compute the `createdAt` span covering a set of webhook delivery friendlyIds, for partition-pruning
26+
* a lookup by id on the RANGE-partitioned `WebhookDelivery` table.
27+
*
28+
* A delivery id body is `base32hex(big-endian ms timestamp then random bytes)`, and base32hex is
29+
* order-preserving, so lexical id order equals chronological order. The earliest and latest
30+
* timestamps therefore sit at the lexical extremes, and we recover the span by decoding only those
31+
* two ids instead of all N. The embedded timestamp equals the row's `createdAt`, so `[gte, lte]`
32+
* covers every row in the set exactly.
33+
*
34+
* Returns `undefined` for an empty set, or if an extreme id fails to decode, so the caller adds no
35+
* `createdAt` predicate and the lookup stays correct (just unpruned).
36+
*/
37+
export function deliveryIdsCreatedAtBounds(
38+
friendlyIds: string[]
39+
): { gte: Date; lte: Date } | undefined {
40+
if (friendlyIds.length === 0) return undefined;
41+
42+
let minBody = WebhookDeliveryId.toId(friendlyIds[0]!);
43+
let maxBody = minBody;
44+
for (let i = 1; i < friendlyIds.length; i++) {
45+
const body = WebhookDeliveryId.toId(friendlyIds[i]!);
46+
if (body < minBody) minBody = body;
47+
if (body > maxBody) maxBody = body;
48+
}
49+
50+
const gte = WebhookDeliveryId.parseTimestamp(minBody);
51+
const lte = WebhookDeliveryId.parseTimestamp(maxBody);
52+
if (!gte || !lte) return undefined;
53+
54+
return { gte, lte };
55+
}

apps/webapp/test/deliveryIdBounds.test.ts

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import { describe, expect, it, vi } from "vitest";
22
import { WebhookDeliveryId } from "@trigger.dev/core/v3/isomorphic";
3-
import { deliveryIdsCreatedAtBounds } from "../app/services/webhookDeliveriesRepository/deliveryIdBounds";
3+
import {
4+
createdAtMsBounds,
5+
deliveryIdsCreatedAtBounds,
6+
} from "../app/services/webhookDeliveriesRepository/deliveryIdBounds";
47

58
function idAt(iso: string): string {
69
vi.setSystemTime(new Date(iso));
@@ -39,14 +42,32 @@ describe("deliveryIdsCreatedAtBounds", () => {
3942
}
4043
});
4144

42-
it("returns undefined when any id is legacy (non-time-encoded), so the caller skips pruning", () => {
43-
vi.useFakeTimers();
44-
try {
45-
const v1 = idAt("2026-08-11T10:00:00.000Z");
46-
expect(deliveryIdsCreatedAtBounds([v1, "whd_legacycuidstyleid"])).toBeUndefined();
47-
expect(deliveryIdsCreatedAtBounds(["whd_legacycuidstyleid"])).toBeUndefined();
48-
} finally {
49-
vi.useRealTimers();
50-
}
45+
it("returns undefined when an id fails to decode, so the caller skips pruning", () => {
46+
expect(deliveryIdsCreatedAtBounds(["whd_notavaliddeliveryid"])).toBeUndefined();
47+
});
48+
});
49+
50+
describe("createdAtMsBounds", () => {
51+
it("returns undefined for an empty set", () => {
52+
expect(createdAtMsBounds([])).toBeUndefined();
53+
});
54+
55+
it("returns a zero-width span for a single value", () => {
56+
const bounds = createdAtMsBounds([1_000]);
57+
expect(bounds?.gte.getTime()).toBe(1_000);
58+
expect(bounds?.lte.getTime()).toBe(1_000);
59+
});
60+
61+
it("spans the smallest and largest value regardless of input order", () => {
62+
const bounds = createdAtMsBounds([50, 10, 30, 90, 40]);
63+
expect(bounds?.gte.getTime()).toBe(10);
64+
expect(bounds?.lte.getTime()).toBe(90);
65+
});
66+
67+
it("handles a large input without a stack overflow (unlike Math.min(...spread))", () => {
68+
const values = Array.from({ length: 300_000 }, (_, i) => i);
69+
const bounds = createdAtMsBounds(values);
70+
expect(bounds?.gte.getTime()).toBe(0);
71+
expect(bounds?.lte.getTime()).toBe(299_999);
5172
});
5273
});

0 commit comments

Comments
 (0)