perf(webapp,run-engine): cut CPU on the engine-facing worker-action routes

Three changes from a CPU profile of the worker-action request path, together
taking on-CPU time per completed run from 9.07ms to 6.59ms (-27%) at ~300 req/s,
with every worker-action p50 down 23-27%.

Split the event-loop monitor in two. The blocked-loop detector installs an
async_hooks hook that fires for every async resource the process creates, and
enabling any async hook also puts V8 on the slow path for promise
instrumentation process-wide; it measured ~14% of on-CPU time plus roughly half
of all GC. It is now opt-in via EVENT_LOOP_MONITOR_ENABLED (default 0). The
event-loop utilization gauge is a single interval timer with no per-request
cost, so it moves to EVENT_LOOP_UTILIZATION_MONITOR_ENABLED (default 1) and
stays on.

Bucket route matching by first static path segment. The existing router patch
removed the per-request re-flatten and regex rebuild, but matching was still a
linear scan over the whole 521-route table. Route-matching self time drops 64%.
Ordering is preserved exactly and equivalence was verified over 20,050
pathnames; apps/webapp/test/routeMatchingPatch.test.ts pins the semantics.

Demote the per-heartbeat and per-dequeue info logs to debug. These are the two
highest-rate engine calls and each wrote a synchronous structured log line on
every request.

Author: ericallam

.server-changes/reduce-webapp-cpu-on-worker-routes.md

@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Cut webapp CPU usage by about a quarter on the routes that workers call most, freeing headroom at the same request rate. Detailed event-loop blocking diagnostics are now off by default (set `EVENT_LOOP_MONITOR_ENABLED=1` to restore them); the event-loop utilization metric is unaffected.

apps/webapp/app/entry.server.tsx

@@ -18,7 +18,7 @@ import type { OperatingSystemPlatform } from "./components/primitives/OperatingS
 import { OperatingSystemContextProvider } from "./components/primitives/OperatingSystemProvider";
 import { assertRunOpsSplitSentinel, Prisma } from "./db.server";
 import { env } from "./env.server";
-import { eventLoopMonitor } from "./eventLoopMonitor.server";
+import { eventLoopMonitor, eventLoopUtilizationMonitor } from "./eventLoopMonitor.server";
 import { logger } from "./services/logger.server";
 import { buildImgSrcDirective, parseCspImageOrigins, withImgSrc } from "./utils/cspImageOrigins";
 import { singleton } from "./utils/singleton";
@@ -360,6 +360,10 @@ if (env.EVENT_LOOP_MONITOR_ENABLED === "1") {
   eventLoopMonitor.enable();
 }
 
+if (env.EVENT_LOOP_UTILIZATION_MONITOR_ENABLED === "1") {
+  eventLoopUtilizationMonitor.enable();
+}
+
 if (remoteBuildsEnabled()) {
   console.log("🏗️  Remote builds enabled");
 } else {

apps/webapp/app/env.server.ts

@@ -995,7 +995,8 @@ const EnvironmentSchema = z
 
     CENTS_PER_RUN: z.coerce.number().default(0),
 
-    EVENT_LOOP_MONITOR_ENABLED: z.string().default("1"),
+    EVENT_LOOP_MONITOR_ENABLED: z.string().default("0"),
+    EVENT_LOOP_UTILIZATION_MONITOR_ENABLED: z.string().default("1"),
     MAXIMUM_LIVE_RELOADING_EVENTS: z.coerce.number().int().default(1000),
     MAXIMUM_TRACE_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(25_000),
     MAXIMUM_TRACE_DETAILED_SUMMARY_VIEW_COUNT: z.coerce.number().int().default(10_000),

apps/webapp/app/eventLoopMonitor.server.ts

@@ -89,25 +89,47 @@ function after(asyncId: number) {
   }
 }
 
+/**
+ * Per-async-resource blocked-loop detection. This is the expensive half: the
+ * hook fires for every async resource the process creates, and enabling any
+ * async hook also puts V8 on the slow path for promise instrumentation
+ * process-wide. On a request-heavy instance it costs roughly a seventh of all
+ * on-CPU time, which is why it is opt-in rather than on by default.
+ */
 export const eventLoopMonitor = singleton("eventLoopMonitor", () => {
   const hook = createHook({ init, before, after, destroy });
 
-  let stopEventLoopUtilizationMonitoring: () => void;
-
   return {
     enable: () => {
       console.log("🥸  Initializing event loop monitor");
 
       hook.enable();
-
-      stopEventLoopUtilizationMonitoring = startEventLoopUtilizationMonitoring();
     },
     disable: () => {
       console.log("🥸  Disabling event loop monitor");
 
       hook.disable();
+    },
+  };
+});
+
+/**
+ * The cheap half: a single interval timer reading `eventLoopUtilization()`.
+ * It costs nothing per request, so it stays on by default and is what a
+ * high-traffic instance should rely on when the async hook is too expensive.
+ */
+export const eventLoopUtilizationMonitor = singleton("eventLoopUtilizationMonitor", () => {
+  let stop: (() => void) | undefined;
 
-      stopEventLoopUtilizationMonitoring?.();
+  return {
+    enable: () => {
+      console.log("🥸  Initializing event loop utilization monitor");
+
+      stop = startEventLoopUtilizationMonitoring();
+    },
+    disable: () => {
+      stop?.();
+      stop = undefined;
     },
   };
 });

apps/webapp/test/routeMatchingPatch.test.ts

@@ -0,0 +1,119 @@
+import { matchRoutes, type RouteObject } from "@remix-run/router";
+import { describe, expect, it } from "vitest";
+
+/**
+ * Guards `patches/@remix-run__router@1.23.3.patch`.
+ *
+ * The patch buckets ranked route branches by their first static path segment so
+ * a request only scans branches that could match it. That is a change to the
+ * matcher's search order, so these cases pin the behaviour that must survive
+ * it: branches whose leading segment is dynamic, splat or optional have to stay
+ * reachable from every pathname, case-insensitive matching has to keep working
+ * across the bucket lookup, and a more specific static route must still beat a
+ * dynamic one.
+ *
+ * If the patch is ever dropped, these should still pass against the stock
+ * matcher — they assert matching semantics, not the optimisation.
+ */
+const routes: RouteObject[] = [
+  {
+    id: "root",
+    path: "/",
+    children: [
+      { id: "index", index: true },
+      { id: "engine-dequeue", path: "engine/v1/worker-actions/dequeue" },
+      {
+        id: "engine-heartbeat",
+        path: "engine/v1/worker-actions/runs/:runFriendlyId/snapshots/:snapshotFriendlyId/heartbeat",
+      },
+      { id: "engine-splat", path: "engine/*" },
+      { id: "api-run", path: "api/v1/runs/:runId" },
+      { id: "api-summary", path: "api/v1/runs/summary" },
+      { id: "case-sensitive", path: "Engine/CaseCheck", caseSensitive: true },
+      { id: "optional-lang", path: ":lang?/docs" },
+      { id: "org-project", path: ":org/projects/:projectId" },
+      { id: "orgs-settings", path: "orgs/settings" },
+      { id: "orgs-dynamic", path: "orgs/:orgSlug" },
+      { id: "catch-all", path: "*" },
+    ],
+  },
+];
+
+function matchedIds(pathname: string, basename?: string): string[] | null {
+  const matches = matchRoutes(routes, pathname, basename);
+  return matches ? matches.map((match) => match.route.id!) : null;
+}
+
+function paramsFor(pathname: string): Record<string, string | undefined> {
+  const matches = matchRoutes(routes, pathname);
+  return matches ? matches[matches.length - 1]!.params : {};
+}
+
+describe("route matching (patched matcher)", () => {
+  it("matches a fully static engine route", () => {
+    expect(matchedIds("/engine/v1/worker-actions/dequeue")).toEqual(["root", "engine-dequeue"]);
+  });
+
+  it("matches a dynamic engine route and extracts params", () => {
+    const pathname = "/engine/v1/worker-actions/runs/run_abc/snapshots/snap_def/heartbeat";
+    expect(matchedIds(pathname)).toEqual(["root", "engine-heartbeat"]);
+    expect(paramsFor(pathname)).toMatchObject({
+      runFriendlyId: "run_abc",
+      snapshotFriendlyId: "snap_def",
+    });
+  });
+
+  it("prefers a static route over a dynamic sibling at the same depth", () => {
+    expect(matchedIds("/api/v1/runs/summary")).toEqual(["root", "api-summary"]);
+    expect(matchedIds("/api/v1/runs/run_abc")).toEqual(["root", "api-run"]);
+    expect(matchedIds("/orgs/settings")).toEqual(["root", "orgs-settings"]);
+    expect(matchedIds("/orgs/acme")).toEqual(["root", "orgs-dynamic"]);
+  });
+
+  it("falls back to a splat within the same first segment", () => {
+    expect(matchedIds("/engine/something/unrouted")).toEqual(["root", "engine-splat"]);
+  });
+
+  it("keeps routes with a dynamic first segment reachable", () => {
+    expect(matchedIds("/acme/projects/proj_1")).toEqual(["root", "org-project"]);
+    expect(paramsFor("/acme/projects/proj_1")).toMatchObject({
+      org: "acme",
+      projectId: "proj_1",
+    });
+  });
+
+  it("keeps routes with an optional first segment reachable both ways", () => {
+    expect(matchedIds("/docs")).toEqual(["root", "optional-lang"]);
+    expect(matchedIds("/en/docs")).toEqual(["root", "optional-lang"]);
+  });
+
+  it("matches case-insensitively by default", () => {
+    expect(matchedIds("/ENGINE/v1/worker-actions/dequeue")).toEqual(["root", "engine-dequeue"]);
+    expect(matchedIds("/API/v1/runs/summary")).toEqual(["root", "api-summary"]);
+  });
+
+  it("honours caseSensitive routes", () => {
+    expect(matchedIds("/Engine/CaseCheck")).toEqual(["root", "case-sensitive"]);
+    expect(matchedIds("/engine/casecheck")).toEqual(["root", "engine-splat"]);
+  });
+
+  it("falls through to the global catch-all for an unknown first segment", () => {
+    expect(matchedIds("/totally/unknown/path")).toEqual(["root", "catch-all"]);
+  });
+
+  it("matches the index route at the root", () => {
+    expect(matchedIds("/")).toEqual(["root", "index"]);
+  });
+
+  it("still strips a basename before matching", () => {
+    expect(matchedIds("/base/engine/v1/worker-actions/dequeue", "/base")).toEqual([
+      "root",
+      "engine-dequeue",
+    ]);
+    expect(matchRoutes(routes, "/elsewhere/engine", "/base")).toBeNull();
+  });
+
+  it("matches a percent-encoded first segment", () => {
+    expect(matchedIds("/%65ngine/v1/worker-actions/dequeue")).toEqual(["root", "engine-dequeue"]);
+  });
+});

internal-packages/run-engine/src/engine/systems/dequeueSystem.ts

@@ -163,7 +163,7 @@ export class DequeueSystem {
             ? Math.max(0, Date.now() - message.message.eligibleAtMs)
             : undefined;
 
-        this.$.logger.info("DequeueSystem.dequeueFromWorkerQueue dequeued message", {
+        this.$.logger.debug("DequeueSystem.dequeueFromWorkerQueue dequeued message", {
           runId,
           orgId,
           environmentId: message.message.environmentId,

internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts

@@ -567,7 +567,7 @@ export class ExecutionSnapshotSystem {
       });
     }
 
-    this.$.logger.info("heartbeatRun snapshot heartbeat updated", {
+    this.$.logger.debug("heartbeatRun snapshot heartbeat updated", {
       id: latestSnapshot.id,
       runId: latestSnapshot.runId,
       lastHeartbeatAt: new Date(),

patches/@remix-run__router@1.23.3.patch

@@ -1,8 +1,8 @@
 diff --git a/dist/router.cjs.js b/dist/router.cjs.js
-index 6aa7db6fb5a7182afcdf17b16a3356abfa1e7945..95edbde228beff8dbd13fb2800302e31a932ef25 100644
+index 6aa7db6fb5a7182afcdf17b16a3356abfa1e7945..5ae0b0e632a356493c3a8b0c88ebd396e8f5305b 100644
 --- a/dist/router.cjs.js
 +++ b/dist/router.cjs.js
-@@ -783,6 +783,11 @@ function convertRoutesToDataRoutes(routes, mapRouteProperties, parentPath, manif
+@@ -783,6 +783,51 @@ function convertRoutesToDataRoutes(routes, mapRouteProperties, parentPath, manif
   *
   * @see https://reactrouter.com/v6/utils/match-routes
   */
@@ -11,10 +11,50 @@ index 6aa7db6fb5a7182afcdf17b16a3356abfa1e7945..95edbde228beff8dbd13fb2800302e31
 +// fix #14967; maintainer suggested patch-package until the Remix 3 route-pattern rewrite).
 +let __branchCache = new WeakMap();
 +let __compileCache = new Map();
++/**
++ * trigger.dev perf patch 2 — bucket ranked branches by their first static path
++ * segment so a request scans only branches that could match it, rather than the
++ * whole 500+ route table. See patches/README.md.
++ */
++let __bucketCache = new WeakMap();
++/**
++ * Returns the lowercased leading segment when it is static, or null when the
++ * branch can match any first segment (dynamic, splat or optional leading
++ * segment, or a root/pathless path) and so must always be considered.
++ */
++function __firstStaticSegment(path) {
++  if (!path || path === "/") return null;
++  let start = path.charCodeAt(0) === 47 ? 1 : 0;
++  let end = path.indexOf("/", start);
++  let seg = end === -1 ? path.slice(start) : path.slice(start, end);
++  if (seg === "") return null;
++  if (seg.indexOf(":") !== -1 || seg.indexOf("*") !== -1 || seg.indexOf("(") !== -1 || seg.indexOf("?") !== -1) {
++    return null;
++  }
++  return seg.toLowerCase();
++}
++function __buildBuckets(branches) {
++  let byFirstSegment = new Map();
++  let always = [];
++  for (let i = 0; i < branches.length; ++i) {
++    let seg = __firstStaticSegment(branches[i].path);
++    if (seg === null) {
++      always.push(i);
++      continue;
++    }
++    let list = byFirstSegment.get(seg);
++    if (!list) {
++      list = [];
++      byFirstSegment.set(seg, list);
++    }
++    list.push(i);
++  }
++  return { byFirstSegment, always };
++}
  function matchRoutes(routes, locationArg, basename) {
    if (basename === void 0) {
      basename = "/";
-@@ -795,8 +800,13 @@ function matchRoutesImpl(routes, locationArg, basename, allowPartial) {
+@@ -795,18 +840,51 @@ function matchRoutesImpl(routes, locationArg, basename, allowPartial) {
    if (pathname == null) {
      return null;
    }
@@ -29,8 +69,54 @@ index 6aa7db6fb5a7182afcdf17b16a3356abfa1e7945..95edbde228beff8dbd13fb2800302e31
 +  }
    let matches = null;
    let decoded = decodePath(pathname);
-   for (let i = 0; matches == null && i < branches.length; ++i) {
-@@ -1115,6 +1125,12 @@ function compilePath(path, caseSensitive, end) {
+-  for (let i = 0; matches == null && i < branches.length; ++i) {
+-    // Incoming pathnames are generally encoded from either window.location
+-    // or from router.navigate, but we want to match against the unencoded
+-    // paths in the route definitions.  Memory router locations won't be
+-    // encoded here but there also shouldn't be anything to decode so this
+-    // should be a safe operation.  This avoids needing matchRoutes to be
+-    // history-aware.
+-    matches = matchRouteBranch(branches[i], decoded, allowPartial);
++  // Incoming pathnames are generally encoded from either window.location
++  // or from router.navigate, but we want to match against the unencoded
++  // paths in the route definitions.  Memory router locations won't be
++  // encoded here but there also shouldn't be anything to decode so this
++  // should be a safe operation.  This avoids needing matchRoutes to be
++  // history-aware.
++  let buckets = __bucketCache.get(branches);
++  if (!buckets) {
++    buckets = __buildBuckets(branches);
++    __bucketCache.set(branches, buckets);
++  }
++  let requestSegment = __firstStaticSegment(decoded);
++  if (requestSegment === null) {
++    for (let i = 0; matches == null && i < branches.length; ++i) {
++      matches = matchRouteBranch(branches[i], decoded, allowPartial);
++    }
++    return matches;
++  }
++  /**
++   * Both lists hold indexes into the already rank-sorted `branches`, so walking
++   * them in ascending-index order preserves the exact evaluation order the
++   * unbucketed scan would have used.
++   */
++  let scoped = buckets.byFirstSegment.get(requestSegment);
++  let always = buckets.always;
++  let si = 0;
++  let ai = 0;
++  let scopedLength = scoped === undefined ? 0 : scoped.length;
++  while (matches == null && (si < scopedLength || ai < always.length)) {
++    let index;
++    if (si < scopedLength && (ai >= always.length || scoped[si] < always[ai])) {
++      index = scoped[si++];
++    } else {
++      index = always[ai++];
++    }
++    matches = matchRouteBranch(branches[index], decoded, allowPartial);
+   }
+   return matches;
+ }
+@@ -1115,6 +1193,12 @@ function compilePath(path, caseSensitive, end) {
    if (end === void 0) {
      end = true;
    }
@@ -43,7 +129,7 @@ index 6aa7db6fb5a7182afcdf17b16a3356abfa1e7945..95edbde228beff8dbd13fb2800302e31
    warning(path === "*" || !path.endsWith("*") || path.endsWith("/*"), "Route path \"" + path + "\" will be treated as if it were " + ("\"" + path.replace(/\*$/, "/*") + "\" because the `*` character must ") + "always follow a `/` in the pattern. To get rid of this warning, " + ("please change the route path to \"" + path.replace(/\*$/, "/*") + "\"."));
    let params = [];
    let regexpSource = "^" + path.replace(/\/*\*?$/, "") // Ignore trailing / and /*, we'll handle it below
-@@ -1147,7 +1163,11 @@ function compilePath(path, caseSensitive, end) {
+@@ -1147,7 +1231,11 @@ function compilePath(path, caseSensitive, end) {
      regexpSource += "(?:(?=\\/|$))";
    } else ;
    let matcher = new RegExp(regexpSource, caseSensitive ? undefined : "i");