fix(supervisor): scrape timeout + schema-level pod-count hysteresis guard

Author: nicktrn

apps/supervisor/src/clients/kubernetes.ts

@@ -60,7 +60,7 @@ export { k8s };
  * One lightweight aggregate read - not a pod listing. Requires the service
  * account to be granted GET on the /metrics non-resource URL.
  */
-export function createApiserverMetricsFetcher(): () => Promise<string> {
+export function createApiserverMetricsFetcher(timeoutMs: number): () => Promise<string> {
   const kubeConfig = getKubeConfig();
 
   return async () => {
@@ -98,6 +98,11 @@ export function createApiserverMetricsFetcher(): () => Promise<string> {
           }
         });
       });
+      // Without this a hung connect/TLS/read never settles, and the monitor's
+      // refreshInFlight guard would freeze the source (silent fail-open).
+      req.setTimeout(timeoutMs, () => {
+        req.destroy(new Error(`apiserver /metrics scrape timed out after ${timeoutMs}ms`));
+      });
       req.on("error", reject);
       req.end();
     });

apps/supervisor/src/env.test.ts

@@ -50,4 +50,15 @@ describe("Env superRefine - backpressure source awareness", () => {
       })
     ).not.toThrow();
   });
+
+  it("rejects pod-count release >= engage when the source is enabled", () => {
+    expect(() =>
+      Env.parse({
+        ...base,
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED: "true",
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE: "100",
+        TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE: "100",
+      })
+    ).toThrow();
+  });
 });

apps/supervisor/src/env.ts

@@ -78,6 +78,13 @@ export const Env = z
     TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE: z.coerce.number().int().positive().default(10_000),
     TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE: z.coerce.number().int().positive().default(5_000),
     TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_REFRESH_MS: z.coerce.number().int().positive().default(5_000),
+    // Hard timeout on the apiserver /metrics scrape. A hung request would otherwise
+    // never settle and freeze the monitor's refresh loop (fail-open silently).
+    TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS: z.coerce
+      .number()
+      .int()
+      .positive()
+      .default(10_000),
 
     // Optional services
     TRIGGER_WARM_START_URL: z.string().optional(),
@@ -317,6 +324,18 @@ export const Env = z
     TRIGGER_WIDE_EVENTS_NOISY_ROUTES: BoolEnv.default(false),
   })
   .superRefine((data, ctx) => {
+    if (
+      data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED &&
+      data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE >=
+        data.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE
+    ) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message:
+          "TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE must be less than TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE",
+        path: ["TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE"],
+      });
+    }
     if (data.COMPUTE_SNAPSHOTS_ENABLED && !data.TRIGGER_METADATA_URL) {
       ctx.addIssue({
         code: z.ZodIssueCode.custom,

apps/supervisor/src/index.ts

@@ -252,14 +252,7 @@ class ManagedSupervisor {
     // Pod-count source (in-process apiserver scrape). Namespaced metrics so the
     // redis source's metric names are preserved.
     if (env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENABLED) {
-      if (
-        env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE >=
-        env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE
-      ) {
-        throw new Error(
-          "TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE must be less than TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE"
-        );
-      }
+      // RELEASE < ENGAGE is enforced in env.ts (superRefine), so it's valid here.
       const podCountGauge = new Gauge({
         name: "supervisor_cluster_pod_count",
         help: "Total pod objects stored in the cluster, scraped for backpressure",
@@ -269,7 +262,9 @@ class ManagedSupervisor {
         new BackpressureMonitor({
           enabled: true,
           source: new K8sPodCountSignalSource({
-            fetchMetrics: createApiserverMetricsFetcher(),
+            fetchMetrics: createApiserverMetricsFetcher(
+              env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_SCRAPE_TIMEOUT_MS
+            ),
             engageThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_ENGAGE,
             releaseThreshold: env.TRIGGER_DEQUEUE_BACKPRESSURE_POD_COUNT_RELEASE,
             reportPodCount: (count) => podCountGauge.set(count),