test: add PromQL smoke tests for MCP across backends and in the image

.github/workflows/docker-build.yml

@@ -148,7 +148,7 @@ jobs:
   # rebuilds the entire App/prod image. To avoid duplicating that ~8-minute
   # Node build on every app change, the pull-request path filter below is
   # scoped ONLY to the incremental bundling inputs (ClickHouse / OTel / the
-  # shared Dockerfile) — NOT api/app/common-utils/deps. Those are covered by
+  # shared Dockerfile) and the smoke script — NOT api/app/common-utils/deps. Those are covered by
   # the App Image job above, and they can't break the bundling layers, which
   # only COPY the prod output. Pushes to main ignore the filter entirely.
   all-in-one-image:
@@ -169,6 +169,7 @@ jobs:
             docker/clickhouse/**
             docker/otel-collector/**
             .vex/**
+            scripts/ci/smoke-all-in-one.sh
       - name: Setup Docker Buildx
         if:
           github.event_name != 'pull_request' ||
@@ -185,6 +186,8 @@ jobs:
           target: all-in-one-auth
           platforms: linux/amd64
           push: false
+          load: true
+          tags: hyperdx-aio-auth-smoke:pr
           build-contexts: |
             clickhouse=./docker/clickhouse
             otel-collector=./docker/otel-collector
@@ -232,3 +235,11 @@ jobs:
           steps.changed-files.outputs.any_changed == 'true'
         timeout-minutes: 10
         run: scripts/ci/smoke-all-in-one.sh hyperdx-aio-smoke:pr
+      # Boot the auth image with PromQL on, remote-write one OTLP gauge into
+      # the TimeSeries table, and query it over MCP with a PromQL tile.
+      - name: Smoke test PromQL over MCP on the auth image
+        if:
+          github.event_name != 'pull_request' ||
+          steps.changed-files.outputs.any_changed == 'true'
+        timeout-minutes: 10
+        run: scripts/ci/smoke-all-in-one.sh hyperdx-aio-auth-smoke:pr --promql-mcp

Makefile

@@ -10,18 +10,20 @@ include .env
 # parallel. scripts/slots.sh computes them; override the slot with
 # `make dev-int HDX_CI_SLOT=5`.
 # ---------------------------------------------------------------------------
-# One shell for all six values: this runs on every make invocation, and each
+# One shell for all seven values: this runs on every make invocation, and each
 # shell costs ~20ms.
-_hdx_ci := $(shell HDX_CI_SLOT='$(HDX_CI_SLOT)' sh -c '. ./scripts/slots.sh && hdx_ci_ports && echo "$$HDX_CI_SLOT $$HDX_CI_PROJECT $$HDX_CI_CH_PORT $$HDX_CI_MONGO_PORT $$HDX_CI_API_PORT $$HDX_CI_OPAMP_PORT"')
+_hdx_ci := $(shell HDX_CI_SLOT='$(HDX_CI_SLOT)' sh -c '. ./scripts/slots.sh && hdx_ci_ports && echo "$$HDX_CI_SLOT $$HDX_CI_PROJECT $$HDX_CI_CH_PORT $$HDX_CI_MONGO_PORT $$HDX_CI_API_PORT $$HDX_CI_OPAMP_PORT $$HDX_CI_PROMETHEUS_PORT"')
 
 HDX_CI_SLOT      := $(word 1,$(_hdx_ci))
 HDX_CI_PROJECT   := $(word 2,$(_hdx_ci))
 HDX_CI_CH_PORT   := $(word 3,$(_hdx_ci))
 HDX_CI_MONGO_PORT:= $(word 4,$(_hdx_ci))
 HDX_CI_API_PORT  := $(word 5,$(_hdx_ci))
 HDX_CI_OPAMP_PORT:= $(word 6,$(_hdx_ci))
+HDX_CI_PROMETHEUS_PORT:= $(word 7,$(_hdx_ci))
 
-export HDX_CI_CH_PORT HDX_CI_MONGO_PORT HDX_CI_API_PORT HDX_CI_OPAMP_PORT
+export HDX_CI_CH_PORT HDX_CI_MONGO_PORT HDX_CI_API_PORT HDX_CI_OPAMP_PORT \
+	HDX_CI_PROMETHEUS_PORT
 
 # Log directory for dev-portal visibility (integration tests)
 HDX_CI_LOGS_DIR := $(HOME)/.config/hyperdx/dev-slots/$(HDX_CI_SLOT)/logs-int
@@ -155,7 +157,7 @@ dev-int-build:
 
 .PHONY: dev-int
 dev-int:
-	@echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))"
+	@echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT) prometheus=$(HDX_CI_PROMETHEUS_PORT))"
 	@mkdir -p $(HDX_CI_LOGS_DIR)
 	@bash scripts/ensure-dev-portal.sh
 	docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d

agent_docs/development.md

@@ -170,18 +170,20 @@ without port conflicts.
 
 Port mapping (base + slot):
 
-| Service         | Default port (slot 0) | Variable          |
-| --------------- | --------------------- | ----------------- |
-| ClickHouse HTTP | 18123                 | HDX_CI_CH_PORT    |
-| MongoDB         | 39999                 | HDX_CI_MONGO_PORT |
-| API test server | 19000                 | HDX_CI_API_PORT   |
-| OpAMP           | 14320                 | HDX_CI_OPAMP_PORT |
+| Service         | Default port (slot 0) | Variable               |
+| --------------- | --------------------- | ---------------------- |
+| ClickHouse HTTP | 18123                 | HDX_CI_CH_PORT         |
+| MongoDB         | 39999                 | HDX_CI_MONGO_PORT      |
+| API test server | 19000                 | HDX_CI_API_PORT        |
+| OpAMP           | 14320                 | HDX_CI_OPAMP_PORT      |
+| Prometheus      | 19100                 | HDX_CI_PROMETHEUS_PORT |
 
 **CI Testing Notes:**
 
 - Uses separate Docker Compose configuration (`docker-compose.ci.yml`)
 - Isolated test environment with unique `-p int-<slot>` project name
-- Includes all necessary services (ClickHouse, MongoDB, OTel Collector)
+- Includes all necessary services (ClickHouse, MongoDB, OTel Collector, and a
+  Prometheus that remote-reads `default.metrics_ts` for PromQL tests)
 - Tests run against real database instances for accurate integration testing
 
 ### E2E Testing

docker-compose.ci.yml

@@ -34,6 +34,16 @@ services:
       - '${HDX_CI_CH_PORT:-18123}:8123' # http api
     networks:
       - internal
+  prometheus:
+    image: prom/prometheus:v3.16.0
+    volumes:
+      - ./docker/prometheus/prometheus.ci.yml:/etc/prometheus/prometheus.yml
+    ports:
+      - '${HDX_CI_PROMETHEUS_PORT:-19100}:9090'
+    networks:
+      - internal
+    depends_on:
+      - ch-server
   db:
     image: mongo:5.0.32-focal
     command: --port 29999

docker/prometheus/prometheus.ci.yml

@@ -0,0 +1,12 @@
+# Integration tests: a Prometheus with no local data that reads every sample
+# from ClickHouse's remote-read handler (default.metrics_ts, see
+# docker/clickhouse/local/config.xml). PromQL tests can then compare Prometheus
+# with ClickHouse's own PromQL engine on the same samples.
+global:
+  scrape_interval: 1h
+
+remote_read:
+  - url: http://ch-server:9363/read
+    # Without this, Prometheus skips the remote for the window its own
+    # (empty) TSDB would cover.
+    read_recent: true

packages/api/.env.test

@@ -1,6 +1,7 @@
 # Environment configuration for Integration tests, corresponding to ports defined in docker-compose.ci.yml
 # For multi-agent worktree support, ports are configured via HDX_CI_* env vars.
-# Set HDX_CI_CH_PORT, HDX_CI_MONGO_PORT, HDX_CI_API_PORT, HDX_CI_OPAMP_PORT
+# Set HDX_CI_CH_PORT, HDX_CI_MONGO_PORT, HDX_CI_API_PORT, HDX_CI_OPAMP_PORT,
+# HDX_CI_PROMETHEUS_PORT
 # before running tests to use non-default ports (see Makefile dev-int target).
 CLICKHOUSE_HOST=http://localhost:${HDX_CI_CH_PORT:-18123}
 CLICKHOUSE_PASSWORD=api
@@ -11,6 +12,8 @@ MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test
 NODE_ENV=test
 PORT=${HDX_CI_API_PORT:-19000}
 OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320}
+# The Prometheus in docker-compose.ci.yml, which remote-reads default.metrics_ts.
+CI_PROMETHEUS_URL=http://localhost:${HDX_CI_PROMETHEUS_PORT:-19100}
 # Default to only logging errors. Adjust if you need more verbosity.
 # Note: the logger module is mocked in jest.setup.ts to suppress expected
 # operational noise (validation errors, MCP tool errors, etc.) during tests.

packages/api/src/fixtures.ts

@@ -289,6 +289,11 @@ export type TimeSeriesFixtureSeries = {
   /** Series window in unix seconds; ignored when the table stores no bounds. */
   startSec: number;
   endSec: number;
+  /**
+   * `[unix seconds, value]` samples. With `withSamples`, written instead of
+   * the value 1 at `startSec` and `endSec`.
+   */
+  samples?: [number, number][];
 };
 
 /**
@@ -392,7 +397,24 @@ export const seedTimeSeriesTagsTable = async ({
       );
     }
   }
-  if (withSamples) {
+  const ownSamples = series.filter(s => s.samples?.length).length;
+  if (ownSamples > 0 && ownSamples < series.length) {
+    throw new Error('give every series its own samples, or none');
+  }
+  if (withSamples && ownSamples > 0) {
+    for (const s of series) {
+      const samples = (s.samples ?? [])
+        .map(([ts, value]) => `(${ts}, ${value})`)
+        .join(', ');
+      await executeTimeSeriesSqlCommand(
+        `INSERT INTO TABLE FUNCTION timeSeriesData('${database}', '${table}')
+         SELECT id, toDateTime64(sample.1, 3) AS timestamp, sample.2 AS value
+         FROM timeSeriesTags('${database}', '${table}')
+         ARRAY JOIN [${samples}] AS sample
+         WHERE metric_name = ${quoted(s.metricName)} AND tags = ${mapLiteral(s.tags)}`,
+      );
+    }
+  } else if (withSamples) {
     const bounds = timeRanges
       ? `(SELECT id, min(min_time) AS min_time, max(max_time) AS max_time
           FROM timeSeriesTimeRanges('${database}', '${table}') GROUP BY id)`