From 147770c7785d82c77a363d6c6593c7449839912b Mon Sep 17 00:00:00 2001 From: Tom Alexander Date: Mon, 9 Nov 2026 14:41:58 -0500 Subject: [PATCH] test: add PromQL smoke tests for MCP across backends and in the image Integration: a Prometheus in docker-compose.ci.yml remote-reads default.metrics_ts from ClickHouse, so PromQL tiles, clickstack_promql and dashboard variables are checked to return the same results from a Prometheus endpoint, ClickHouse's Prometheus API and the table function, on one seeded dataset. A second suite covers the 10,000-row table cap, the limit param and the response budget with many series; line and reduced tiles still go over budget because seriesLabels is not trimmed, so those cases are it.failing until that is fixed. Image: smoke-all-in-one.sh gains --promql-mcp, which boots the auth all-in-one image with PromQL on, remote-writes one OTLP gauge and queries it over MCP. docker-build.yml runs it on the auth image. --- .github/workflows/docker-build.yml | 13 +- Makefile | 10 +- agent_docs/development.md | 16 +- docker-compose.ci.yml | 10 + docker/prometheus/prometheus.ci.yml | 12 + packages/api/.env.test | 5 +- packages/api/src/fixtures.ts | 24 +- .../mcp/__tests__/promqlBackends.int.test.ts | 426 ++++++++++++++++++ .../__tests__/promqlLargeResults.int.test.ts | 155 +++++++ scripts/__tests__/slots.test.mjs | 1 + scripts/ci/smoke-all-in-one.sh | 99 +++- scripts/slots.sh | 5 +- 12 files changed, 757 insertions(+), 19 deletions(-) create mode 100644 docker/prometheus/prometheus.ci.yml create mode 100644 packages/api/src/mcp/__tests__/promqlBackends.int.test.ts create mode 100644 packages/api/src/mcp/__tests__/promqlLargeResults.int.test.ts diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 513261b0d1..ed29a14d13 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -148,7 +148,7 @@ jobs: # rebuilds the entire App/prod image. To avoid duplicating that ~8-minute # Node build on every app change, the pull-request path filter below is # scoped ONLY to the incremental bundling inputs (ClickHouse / OTel / the - # shared Dockerfile) — NOT api/app/common-utils/deps. Those are covered by + # shared Dockerfile) and the smoke script — NOT api/app/common-utils/deps. Those are covered by # the App Image job above, and they can't break the bundling layers, which # only COPY the prod output. Pushes to main ignore the filter entirely. all-in-one-image: @@ -169,6 +169,7 @@ jobs: docker/clickhouse/** docker/otel-collector/** .vex/** + scripts/ci/smoke-all-in-one.sh - name: Setup Docker Buildx if: github.event_name != 'pull_request' || @@ -185,6 +186,8 @@ jobs: target: all-in-one-auth platforms: linux/amd64 push: false + load: true + tags: hyperdx-aio-auth-smoke:pr build-contexts: | clickhouse=./docker/clickhouse otel-collector=./docker/otel-collector @@ -232,3 +235,11 @@ jobs: steps.changed-files.outputs.any_changed == 'true' timeout-minutes: 10 run: scripts/ci/smoke-all-in-one.sh hyperdx-aio-smoke:pr + # Boot the auth image with PromQL on, remote-write one OTLP gauge into + # the TimeSeries table, and query it over MCP with a PromQL tile. + - name: Smoke test PromQL over MCP on the auth image + if: + github.event_name != 'pull_request' || + steps.changed-files.outputs.any_changed == 'true' + timeout-minutes: 10 + run: scripts/ci/smoke-all-in-one.sh hyperdx-aio-auth-smoke:pr --promql-mcp diff --git a/Makefile b/Makefile index c682bacfc9..2f7bc1e19f 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,9 @@ include .env # parallel. scripts/slots.sh computes them; override the slot with # `make dev-int HDX_CI_SLOT=5`. # --------------------------------------------------------------------------- -# One shell for all six values: this runs on every make invocation, and each +# One shell for all seven values: this runs on every make invocation, and each # shell costs ~20ms. -_hdx_ci := $(shell HDX_CI_SLOT='$(HDX_CI_SLOT)' sh -c '. ./scripts/slots.sh && hdx_ci_ports && echo "$$HDX_CI_SLOT $$HDX_CI_PROJECT $$HDX_CI_CH_PORT $$HDX_CI_MONGO_PORT $$HDX_CI_API_PORT $$HDX_CI_OPAMP_PORT"') +_hdx_ci := $(shell HDX_CI_SLOT='$(HDX_CI_SLOT)' sh -c '. ./scripts/slots.sh && hdx_ci_ports && echo "$$HDX_CI_SLOT $$HDX_CI_PROJECT $$HDX_CI_CH_PORT $$HDX_CI_MONGO_PORT $$HDX_CI_API_PORT $$HDX_CI_OPAMP_PORT $$HDX_CI_PROMETHEUS_PORT"') HDX_CI_SLOT := $(word 1,$(_hdx_ci)) HDX_CI_PROJECT := $(word 2,$(_hdx_ci)) @@ -20,8 +20,10 @@ HDX_CI_CH_PORT := $(word 3,$(_hdx_ci)) HDX_CI_MONGO_PORT:= $(word 4,$(_hdx_ci)) HDX_CI_API_PORT := $(word 5,$(_hdx_ci)) HDX_CI_OPAMP_PORT:= $(word 6,$(_hdx_ci)) +HDX_CI_PROMETHEUS_PORT:= $(word 7,$(_hdx_ci)) -export HDX_CI_CH_PORT HDX_CI_MONGO_PORT HDX_CI_API_PORT HDX_CI_OPAMP_PORT +export HDX_CI_CH_PORT HDX_CI_MONGO_PORT HDX_CI_API_PORT HDX_CI_OPAMP_PORT \ + HDX_CI_PROMETHEUS_PORT # Log directory for dev-portal visibility (integration tests) HDX_CI_LOGS_DIR := $(HOME)/.config/hyperdx/dev-slots/$(HDX_CI_SLOT)/logs-int @@ -155,7 +157,7 @@ dev-int-build: .PHONY: dev-int dev-int: - @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT))" + @echo "Using CI slot $(HDX_CI_SLOT) (project=$(HDX_CI_PROJECT) ch=$(HDX_CI_CH_PORT) mongo=$(HDX_CI_MONGO_PORT) api=$(HDX_CI_API_PORT) prometheus=$(HDX_CI_PROMETHEUS_PORT))" @mkdir -p $(HDX_CI_LOGS_DIR) @bash scripts/ensure-dev-portal.sh docker compose -p $(HDX_CI_PROJECT) -f ./docker-compose.ci.yml up -d diff --git a/agent_docs/development.md b/agent_docs/development.md index 305c0d9380..5ada97c3e0 100644 --- a/agent_docs/development.md +++ b/agent_docs/development.md @@ -170,18 +170,20 @@ without port conflicts. Port mapping (base + slot): -| Service | Default port (slot 0) | Variable | -| --------------- | --------------------- | ----------------- | -| ClickHouse HTTP | 18123 | HDX_CI_CH_PORT | -| MongoDB | 39999 | HDX_CI_MONGO_PORT | -| API test server | 19000 | HDX_CI_API_PORT | -| OpAMP | 14320 | HDX_CI_OPAMP_PORT | +| Service | Default port (slot 0) | Variable | +| --------------- | --------------------- | ---------------------- | +| ClickHouse HTTP | 18123 | HDX_CI_CH_PORT | +| MongoDB | 39999 | HDX_CI_MONGO_PORT | +| API test server | 19000 | HDX_CI_API_PORT | +| OpAMP | 14320 | HDX_CI_OPAMP_PORT | +| Prometheus | 19100 | HDX_CI_PROMETHEUS_PORT | **CI Testing Notes:** - Uses separate Docker Compose configuration (`docker-compose.ci.yml`) - Isolated test environment with unique `-p int-` project name -- Includes all necessary services (ClickHouse, MongoDB, OTel Collector) +- Includes all necessary services (ClickHouse, MongoDB, OTel Collector, and a + Prometheus that remote-reads `default.metrics_ts` for PromQL tests) - Tests run against real database instances for accurate integration testing ### E2E Testing diff --git a/docker-compose.ci.yml b/docker-compose.ci.yml index 814e45980f..282f899e27 100644 --- a/docker-compose.ci.yml +++ b/docker-compose.ci.yml @@ -34,6 +34,16 @@ services: - '${HDX_CI_CH_PORT:-18123}:8123' # http api networks: - internal + prometheus: + image: prom/prometheus:v3.16.0 + volumes: + - ./docker/prometheus/prometheus.ci.yml:/etc/prometheus/prometheus.yml + ports: + - '${HDX_CI_PROMETHEUS_PORT:-19100}:9090' + networks: + - internal + depends_on: + - ch-server db: image: mongo:5.0.32-focal command: --port 29999 diff --git a/docker/prometheus/prometheus.ci.yml b/docker/prometheus/prometheus.ci.yml new file mode 100644 index 0000000000..bbbb15b453 --- /dev/null +++ b/docker/prometheus/prometheus.ci.yml @@ -0,0 +1,12 @@ +# Integration tests: a Prometheus with no local data that reads every sample +# from ClickHouse's remote-read handler (default.metrics_ts, see +# docker/clickhouse/local/config.xml). PromQL tests can then compare Prometheus +# with ClickHouse's own PromQL engine on the same samples. +global: + scrape_interval: 1h + +remote_read: + - url: http://ch-server:9363/read + # Without this, Prometheus skips the remote for the window its own + # (empty) TSDB would cover. + read_recent: true diff --git a/packages/api/.env.test b/packages/api/.env.test index 45ee1e9b43..7d42d55f12 100644 --- a/packages/api/.env.test +++ b/packages/api/.env.test @@ -1,6 +1,7 @@ # Environment configuration for Integration tests, corresponding to ports defined in docker-compose.ci.yml # For multi-agent worktree support, ports are configured via HDX_CI_* env vars. -# Set HDX_CI_CH_PORT, HDX_CI_MONGO_PORT, HDX_CI_API_PORT, HDX_CI_OPAMP_PORT +# Set HDX_CI_CH_PORT, HDX_CI_MONGO_PORT, HDX_CI_API_PORT, HDX_CI_OPAMP_PORT, +# HDX_CI_PROMETHEUS_PORT # before running tests to use non-default ports (see Makefile dev-int target). CLICKHOUSE_HOST=http://localhost:${HDX_CI_CH_PORT:-18123} CLICKHOUSE_PASSWORD=api @@ -11,6 +12,8 @@ MONGO_URI=mongodb://localhost:${HDX_CI_MONGO_PORT:-39999}/hyperdx-test NODE_ENV=test PORT=${HDX_CI_API_PORT:-19000} OPAMP_PORT=${HDX_CI_OPAMP_PORT:-14320} +# The Prometheus in docker-compose.ci.yml, which remote-reads default.metrics_ts. +CI_PROMETHEUS_URL=http://localhost:${HDX_CI_PROMETHEUS_PORT:-19100} # Default to only logging errors. Adjust if you need more verbosity. # Note: the logger module is mocked in jest.setup.ts to suppress expected # operational noise (validation errors, MCP tool errors, etc.) during tests. diff --git a/packages/api/src/fixtures.ts b/packages/api/src/fixtures.ts index eeb71459f3..097de6f32d 100644 --- a/packages/api/src/fixtures.ts +++ b/packages/api/src/fixtures.ts @@ -289,6 +289,11 @@ export type TimeSeriesFixtureSeries = { /** Series window in unix seconds; ignored when the table stores no bounds. */ startSec: number; endSec: number; + /** + * `[unix seconds, value]` samples. With `withSamples`, written instead of + * the value 1 at `startSec` and `endSec`. + */ + samples?: [number, number][]; }; /** @@ -392,7 +397,24 @@ export const seedTimeSeriesTagsTable = async ({ ); } } - if (withSamples) { + const ownSamples = series.filter(s => s.samples?.length).length; + if (ownSamples > 0 && ownSamples < series.length) { + throw new Error('give every series its own samples, or none'); + } + if (withSamples && ownSamples > 0) { + for (const s of series) { + const samples = (s.samples ?? []) + .map(([ts, value]) => `(${ts}, ${value})`) + .join(', '); + await executeTimeSeriesSqlCommand( + `INSERT INTO TABLE FUNCTION timeSeriesData('${database}', '${table}') + SELECT id, toDateTime64(sample.1, 3) AS timestamp, sample.2 AS value + FROM timeSeriesTags('${database}', '${table}') + ARRAY JOIN [${samples}] AS sample + WHERE metric_name = ${quoted(s.metricName)} AND tags = ${mapLiteral(s.tags)}`, + ); + } + } else if (withSamples) { const bounds = timeRanges ? `(SELECT id, min(min_time) AS min_time, max(max_time) AS max_time FROM timeSeriesTimeRanges('${database}', '${table}') GROUP BY id)` diff --git a/packages/api/src/mcp/__tests__/promqlBackends.int.test.ts b/packages/api/src/mcp/__tests__/promqlBackends.int.test.ts new file mode 100644 index 0000000000..c650eb7736 --- /dev/null +++ b/packages/api/src/mcp/__tests__/promqlBackends.int.test.ts @@ -0,0 +1,426 @@ +import { Metadata } from '@hyperdx/common-utils/dist/core/metadata'; +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { + DEFAULT_DATABASE, + dropTimeSeriesTable, + seedTimeSeriesTagsTable, + TimeSeriesFixtureSeries, +} from '@/fixtures'; +import { setupDashboardTests } from '@/mcp/__tests__/dashboards/setup'; +import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +const mockFetch = jest.mocked(global.fetch); +// Set by jest.setup.ts before it stubs `fetch`. +const realFetch: typeof fetch = globalThis.realFetch; + +// The CI Prometheus remote-reads this table from ClickHouse (see +// docker/prometheus/prometheus.ci.yml), so every backend reads the same samples. +const TABLE = 'metrics_ts'; +const PROMETHEUS_URL = process.env.CI_PROMETHEUS_URL ?? ''; + +const START = 1700000000; +const STEP = 15; +const SAMPLES = 121; // 30 minutes + +const series = ( + metricName: string, + tags: Record, + value: (i: number) => number, +): TimeSeriesFixtureSeries => ({ + metricName, + tags, + startSec: START, + endSec: START + (SAMPLES - 1) * STEP, + samples: Array.from({ length: SAMPLES }, (_, i) => [ + START + i * STEP, + value(i), + ]), +}); + +const SERIES = [ + series('smoke_requests_total', { job: 'checkout', instance: 'c-0' }, i => i), + series( + 'smoke_requests_total', + { job: 'checkout', instance: 'c-1' }, + i => 2 * i, + ), + series('smoke_requests_total', { job: 'cart', instance: 'k-0' }, i => 3 * i), + series('smoke_queue_depth', { job: 'checkout' }, i => i % 10), + ...( + [ + ['0.1', 1], + ['1', 3], + ['+Inf', 4], + ] as const + ).map(([le, perSample]) => + series( + 'smoke_latency_seconds_bucket', + { job: 'checkout', le }, + i => perSample * i, + ), + ), +]; + +// Off the minute, and inside the data once aligned to the granularity. +const startTime = new Date((START + 307) * 1000).toISOString(); +const endTime = new Date((START + 1500) * 1000).toISOString(); + +const RATE = 'rate(smoke_requests_total[1m])'; + +/** Tiles in clickstack_save_dashboard's shape, without sourceId. */ +const TILES: { name: string; config: Record }[] = [ + { + name: 'line by instance', + config: { + displayType: 'line', + expressions: [{ expression: `sum by (instance) (${RATE})` }], + legendTemplate: '{{instance}}', + }, + }, + { + name: 'line with two expressions', + config: { + displayType: 'line', + expressions: [ + { expression: `sum by (job) (${RATE})` }, + { expression: `sum(${RATE})`, alias: 'Total' }, + ], + legendTemplate: '{{job}}', + }, + }, + { + name: 'line, unaligned', + config: { + displayType: 'line', + expressions: [{ expression: 'smoke_queue_depth' }], + granularity: '1 minute', + alignDateRangeToGranularity: false, + }, + }, + { + name: 'stacked bar', + config: { + displayType: 'stacked_bar', + expressions: [{ expression: `sum by (job) (${RATE})` }], + legendTemplate: '{{job}}', + }, + }, + ...(['max', 'min', 'mean', undefined] as const).map(reducer => ({ + name: `number, ${reducer ?? 'default'} reducer`, + config: { + displayType: 'number', + expressions: [{ expression: 'sum(smoke_queue_depth)', reducer }], + }, + })), + { + name: 'number, instant', + config: { + displayType: 'number', + expressions: [ + { expression: 'count(smoke_requests_total)', queryType: 'instant' }, + ], + }, + }, + { + name: 'pie by job', + config: { + displayType: 'pie', + expressions: [{ expression: `sum by (job) (${RATE})` }], + legendTemplate: '{{job}}', + }, + }, + { + name: 'bar by instance, instant', + config: { + displayType: 'bar', + expressions: [ + { expression: 'smoke_requests_total', queryType: 'instant' }, + ], + legendTemplate: '{{instance}}', + }, + }, + { + name: 'table, range', + config: { + displayType: 'table', + expressions: [{ expression: 'smoke_queue_depth' }], + granularity: '5 minute', + }, + }, + { + name: 'table, instant', + config: { + displayType: 'table', + expressions: [ + { expression: 'smoke_requests_total', queryType: 'instant' }, + ], + }, + }, + { + name: 'histogram heatmap', + config: { + displayType: 'heatmap', + heatmapMode: 'histogram', + expressions: [ + { + expression: + 'sum by (le) (rate(smoke_latency_seconds_bucket[$__rate_interval]))', + }, + ], + }, + }, +]; + +type Backend = 'prometheus' | 'clickhouse-http' | 'table-function'; +const BACKENDS: Backend[] = ['prometheus', 'clickhouse-http', 'table-function']; + +describe('MCP PromQL on every backend', () => { + const ctx = setupDashboardTests(); + + beforeAll(async () => { + expect(PROMETHEUS_URL).not.toBe(''); + await seedTimeSeriesTagsTable({ + table: TABLE, + series: SERIES, + withSamples: true, + }); + }); + + afterAll(async () => { + await dropTimeSeriesTable({ table: TABLE }); + }); + + beforeEach(async () => { + await ctx.reconnect({ promqlEnabled: true }); + mockFetch.mockImplementation(realFetch); + }); + + afterEach(() => { + jest.restoreAllMocks(); + mockFetch.mockReset(); + }); + + const createSource = async (backend: Backend) => { + const connection = + backend === 'prometheus' + ? await Connection.create({ + team: ctx.team._id, + name: 'Prometheus', + host: PROMETHEUS_URL, + username: '', + password: '', + isPrometheusEndpoint: true, + }) + : ctx.connection; + const source = await Source.create({ + kind: SourceKind.Promql, + team: ctx.team._id, + connection: connection._id, + name: backend, + from: { databaseName: DEFAULT_DATABASE, tableName: TABLE }, + timestampValueExpression: 'timestamp', + }); + return source._id.toString(); + }; + + /** + * Runs `run` against one backend, and checks its PromQL queries went there. + * ClickHouse before 26.6 has no Prometheus HTTP API, so MCP uses the table + * function. + */ + const onBackend = async (backend: Backend, run: () => Promise) => { + const spy = + backend === 'table-function' + ? jest + .spyOn(Metadata.prototype, 'getServerVersion') + .mockResolvedValue([26, 5, 0, 0]) + : undefined; + const callsBefore = mockFetch.mock.calls.length; + try { + return await run(); + } finally { + spy?.mockRestore(); + const queried = mockFetch.mock.calls + .slice(callsBefore) + .map(([url]) => String(url)) + .filter(url => /\/api\/v1\/query(_range)?\?/.test(url)); + const expected = { + prometheus: `${PROMETHEUS_URL}/api/v1/query`, + 'clickhouse-http': '/prometheus/api/v1/query', + 'table-function': undefined, + }[backend]; + if (expected) { + expect(queried.length).toBeGreaterThan(0); + for (const url of queried) expect(url).toContain(expected); + } else { + expect(queried).toEqual([]); + } + } + }; + + /** In sequence: the table-function backend patches a shared prototype. */ + const onEveryBackend = async (run: (backend: Backend) => Promise) => { + const results: T[] = []; + for (const backend of BACKENDS) results.push(await run(backend)); + return results; + }; + + const call = async (tool: string, args: Record) => { + const result = await callTool(ctx.client!, tool, args); + const text = getFirstText(result); + if (result.isError) throw new Error(`${tool} failed: ${text}`); + return JSON.parse(text); + }; + + /** Saves TILES on one backend's source and returns each tile's result, by name. */ + const queryTiles = async (backend: Backend) => { + const sourceId = await createSource(backend); + const dashboard = await call('clickstack_save_dashboard', { + name: `PromQL on ${backend}`, + tiles: TILES.map(({ name, config }) => ({ + name, + config: { configType: 'promql', sourceId, ...config }, + })), + }); + const results: Record = {}; + for (const tile of dashboard.tiles) { + results[tile.name] = await onBackend(backend, async () => { + const { result } = await call('clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: tile.id, + startTime, + endTime, + }); + return result; + }); + } + return results; + }; + + it('returns the same tile results from every backend', async () => { + const [prometheus, ...clickhouse] = await onEveryBackend(queryTiles); + + for (const name of TILES.map(t => t.name)) { + expect(prometheus[name]).toMatchObject({ rows: expect.any(Number) }); + for (const other of clickhouse) { + expect({ name, result: other[name] }).toEqual({ + name, + result: prometheus[name], + }); + } + } + }); + + it('returns the expected values', async () => { + const results = await queryTiles('prometheus'); + const rows = (name: string) => + (results[name] as { data: Record[] }).data; + + // c-0 counts 1 per 15s sample, c-1 2 and k-0 3. + const c0 = rows('line by instance').find( + row => + row.series_name === 'c-0' && + row.__hdx_time_bucket === '2023-11-14T22:20:00.000Z', + ); + expect(c0?.value).toBeCloseTo(1 / 15); + expect(rows('number, max reducer')).toEqual([ + { series_name: 'sum(smoke_queue_depth)', value: 9 }, + ]); + expect(rows('number, min reducer')).toEqual([ + { series_name: 'sum(smoke_queue_depth)', value: 0 }, + ]); + expect(rows('number, instant')).toEqual([ + { series_name: 'count(smoke_requests_total)', value: 3 }, + ]); + expect(rows('table, instant')).toContainEqual({ + __name__: 'smoke_requests_total', + instance: 'k-0', + job: 'cart', + Value: 300, + }); + expect(new Set(rows('histogram heatmap').map(row => row.le))).toEqual( + new Set(['0.1', '1', '+Inf']), + ); + }); + + it('returns the same clickstack_promql results from every backend', async () => { + const run = (backend: Backend) => + onBackend(backend, async () => { + const sourceId = await createSource(backend); + const args = { sourceId, startTime, endTime }; + return { + range: await call('clickstack_promql', { + ...args, + expression: `sum by (job) (${RATE})`, + }), + instant: await call('clickstack_promql', { + ...args, + expression: 'smoke_queue_depth', + queryType: 'instant', + }), + }; + }); + + const [prometheus, ...clickhouse] = await onEveryBackend(run); + + expect(prometheus.range.result.data.length).toBeGreaterThan(0); + expect(prometheus.instant.result.data).toEqual([ + { __name__: 'smoke_queue_depth', job: 'checkout', Value: 0 }, + ]); + for (const other of clickhouse) expect(other).toEqual(prometheus); + }); + + it('applies a dashboard variable on every backend', async () => { + const run = async (backend: Backend) => { + const sourceId = await createSource(backend); + const dashboard = await call('clickstack_save_dashboard', { + name: `Variables on ${backend}`, + tiles: [ + { + name: 'by instance', + config: { + configType: 'promql', + displayType: 'line', + sourceId, + expressions: [ + { + expression: `sum by (instance) (rate(smoke_requests_total{job=~"$job"}[1m]))`, + }, + ], + legendTemplate: '{{instance}}', + }, + }, + ], + filters: [ + { + type: 'PROMETHEUS_LABEL', + name: 'Job', + sourceId, + label: 'job', + variableName: 'job', + }, + ], + }); + return onBackend(backend, () => + call('clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime, + endTime, + variableValues: [{ name: 'job', values: ['checkout'] }], + }), + ); + }; + + const [prometheus, ...clickhouse] = await onEveryBackend(run); + + expect(Object.keys(prometheus.result.seriesLabels).sort()).toEqual([ + 'c-0', + 'c-1', + ]); + for (const other of clickhouse) expect(other).toEqual(prometheus); + }); +}); diff --git a/packages/api/src/mcp/__tests__/promqlLargeResults.int.test.ts b/packages/api/src/mcp/__tests__/promqlLargeResults.int.test.ts new file mode 100644 index 0000000000..5aa6a752b0 --- /dev/null +++ b/packages/api/src/mcp/__tests__/promqlLargeResults.int.test.ts @@ -0,0 +1,155 @@ +import { SourceKind } from '@hyperdx/common-utils/dist/types'; + +import { setupDashboardTests } from '@/mcp/__tests__/dashboards/setup'; +import { callTool, getFirstText } from '@/mcp/__tests__/mcpTestUtils'; +import Connection from '@/models/connection'; +import { Source } from '@/models/source'; + +const mockFetch = jest.mocked(global.fetch); + +const START = 1700000000; +const startTime = new Date(START * 1000).toISOString(); +const endTime = new Date((START + 3600) * 1000).toISOString(); + +// More series than a table tile keeps (10,000), each a few hundred chars, so +// every response is far over MCP's 50,000-char budget before trimming. +const SERIES = 12_000; +const MAX_RESPONSE_CHARS = 50_000; + +/** The size the trimmer budgets: the compact `{ result }`, not the printed response. */ +const budgetedChars = (text: string) => + JSON.stringify({ result: JSON.parse(text).result }).length; + +const metric = (i: number) => ({ + __name__: 'smoke_up', + job: 'checkout', + instance: `checkout-${i}`, +}); + +/** A Prometheus that answers every query with SERIES series. */ +const answerWithManySeries = () => + mockFetch.mockImplementation(async input => { + const url = new URL(String(input)); + const data = url.pathname.endsWith('/query_range') + ? { + resultType: 'matrix', + result: Array.from({ length: SERIES }, (_, i) => ({ + metric: metric(i), + values: [[START, '1']], + })), + } + : { + resultType: 'vector', + result: Array.from({ length: SERIES }, (_, i) => ({ + metric: metric(i), + value: [START + 3600, String(i)], + })), + }; + return new Response(JSON.stringify({ status: 'success', data })); + }); + +describe('MCP PromQL tiles with large results', () => { + const ctx = setupDashboardTests(); + + beforeEach(async () => { + await ctx.reconnect({ promqlEnabled: true }); + answerWithManySeries(); + }); + + afterEach(() => { + mockFetch.mockReset(); + }); + + const queryTile = async (config: Record) => { + const connection = await Connection.create({ + team: ctx.team._id, + name: 'Prometheus', + host: 'http://prom.example.com', + username: '', + password: '', + isPrometheusEndpoint: true, + }); + const source = await Source.create({ + kind: SourceKind.Promql, + team: ctx.team._id, + connection: connection._id, + name: 'Prometheus', + from: { databaseName: 'prometheus', tableName: 'prometheus' }, + timestampValueExpression: 'timestamp', + }); + const saved = await callTool(ctx.client!, 'clickstack_save_dashboard', { + name: 'Large PromQL results', + tiles: [ + { + name: 'Tile', + config: { + configType: 'promql', + sourceId: source._id.toString(), + ...config, + }, + }, + ], + }); + expect(saved.isError ? getFirstText(saved) : '').toBe(''); + const dashboard = JSON.parse(getFirstText(saved)); + const result = await callTool(ctx.client!, 'clickstack_query_tile', { + dashboardId: dashboard.id, + tileId: dashboard.tiles[0].id, + startTime, + endTime, + }); + expect(result.isError ? getFirstText(result) : '').toBe(''); + return getFirstText(result); + }; + + const requestedUrls = () => + mockFetch.mock.calls.map(([url]) => new URL(String(url))); + + it('caps an instant table at 10,000 rows, asking Prometheus for no more', async () => { + const text = await queryTile({ + displayType: 'table', + expressions: [{ expression: 'smoke_up', queryType: 'instant' }], + }); + + expect(requestedUrls().map(url => url.searchParams.get('limit'))).toEqual([ + '10000', + ]); + const response = JSON.parse(text); + expect(response.result.rows).toBe(10_000); + expect(response.result.data[0]).toEqual({ ...metric(0), Value: 0 }); + expect(response.note).toContain('trimmed'); + expect(budgetedChars(text)).toBeLessThanOrEqual(MAX_RESPONSE_CHARS); + }); + + it('caps a range table at 10,000 rows', async () => { + const text = await queryTile({ + displayType: 'table', + expressions: [{ expression: 'smoke_up' }], + }); + + expect(requestedUrls()[0].searchParams.has('limit')).toBe(false); + expect(JSON.parse(text).result.rows).toBe(10_000); + expect(budgetedChars(text)).toBeLessThanOrEqual(MAX_RESPONSE_CHARS); + }); + + // Known bug: line and reduced tiles list every series' labels in + // seriesLabels, and the trimmer cuts `data` but keeps seriesLabels whole, so + // the response stays over budget. Turn into `it` once seriesLabels is + // trimmed along with data. + it.failing.each(['line', 'number', 'pie', 'bar'])( + 'trims a %s tile with many series to the response budget', + async displayType => { + const text = await queryTile({ + displayType, + expressions: [ + { + expression: 'smoke_up', + ...(displayType === 'line' ? {} : { queryType: 'instant' }), + }, + ], + }); + + expect(budgetedChars(text)).toBeLessThanOrEqual(MAX_RESPONSE_CHARS); + }, + ); +}); diff --git a/scripts/__tests__/slots.test.mjs b/scripts/__tests__/slots.test.mjs index 794bc7ff10..8bb6c8218b 100644 --- a/scripts/__tests__/slots.test.mjs +++ b/scripts/__tests__/slots.test.mjs @@ -75,6 +75,7 @@ test('exports the integration test ports', () => { HDX_CI_MONGO_PORT: '40043', HDX_CI_API_PORT: '19044', HDX_CI_OPAMP_PORT: '14364', + HDX_CI_PROMETHEUS_PORT: '19144', HDX_CI_PROJECT: 'int-44', }); }); diff --git a/scripts/ci/smoke-all-in-one.sh b/scripts/ci/smoke-all-in-one.sh index 45badcb2d3..90a8b7618b 100755 --- a/scripts/ci/smoke-all-in-one.sh +++ b/scripts/ci/smoke-all-in-one.sh @@ -11,16 +11,25 @@ # OTLP receiver (see docker/hyperdx/entry.local.base.sh). A keyless OTLP canary # cannot work there, so instead: boot, wait for /api/health, and assert # ClickHouse is up inside the container. The OTLP port is not mapped in this mode. +# +# --promql-mcp mode — for the auth all-in-one image with PromQL on: boot with an +# INGESTION_API_KEY, push one OTLP gauge through the bundled collector's +# Prometheus remote-write into default.metrics_ts, register a user, add a +# PromQL source, and query the gauge over MCP with clickstack_promql and a saved +# PromQL tile. Needs jq. set -euo pipefail -IMAGE="${1:?usage: smoke-all-in-one.sh [--health-only]}" +IMAGE="${1:?usage: smoke-all-in-one.sh [--health-only|--promql-mcp]}" MODE="${2:-full}" +HEALTH_ONLY=0 +PROMQL_MCP=0 case "$MODE" in --health-only) HEALTH_ONLY=1 ;; - full) HEALTH_ONLY=0 ;; + --promql-mcp) PROMQL_MCP=1 ;; + full) ;; *) - echo "::error::unknown mode '$MODE' (expected --health-only or nothing)" + echo "::error::unknown mode '$MODE' (expected --health-only, --promql-mcp or nothing)" exit 2 ;; esac @@ -29,8 +38,11 @@ NAME="hdx-smoke-$$" APP_PORT="${SMOKE_APP_PORT:-18080}" OTLP_PORT="${SMOKE_OTLP_PORT:-14318}" +COOKIES="$(mktemp)" + cleanup() { docker rm -f "$NAME" >/dev/null 2>&1 || true + rm -f "$COOKIES" } trap cleanup EXIT @@ -51,8 +63,12 @@ else done fi +INGESTION_KEY="smoke-ingestion-key-$$" if [ "$HEALTH_ONLY" = 1 ]; then docker run -d --name "$NAME" -p "$APP_PORT:8080" "$IMAGE" +elif [ "$PROMQL_MCP" = 1 ]; then + docker run -d --name "$NAME" -p "$APP_PORT:8080" -p "$OTLP_PORT:4318" \ + -e ENABLE_PROMQL=true -e INGESTION_API_KEY="$INGESTION_KEY" "$IMAGE" else docker run -d --name "$NAME" -p "$APP_PORT:8080" -p "$OTLP_PORT:4318" "$IMAGE" fi @@ -87,6 +103,83 @@ if [ "$HEALTH_ONLY" = 1 ]; then done fi +if [ "$PROMQL_MCP" = 1 ]; then + API="http://localhost:$APP_PORT/api" + CANARY="smoke-promql-$(date +%s)-$$" + # To stderr: fail also runs inside $(mcp ...), which captures stdout. + fail() { + echo "::error::$1" >&2; docker logs "$NAME" --tail 200 >&2; exit 1 + } + + # Registering also provisions the team's default "Local ClickHouse" connection. + curl -fsS -c "$COOKIES" -X POST "$API/register/password" \ + -H 'Content-Type: application/json' \ + -d '{"email":"smoke@example.com","password":"Smoke-Password-1!","confirmPassword":"Smoke-Password-1!"}' \ + >/dev/null || fail "registration failed" + curl -sS -b "$COOKIES" -c "$COOKIES" -o /dev/null -X POST "$API/login/password" \ + -H 'Content-Type: application/json' \ + -d '{"email":"smoke@example.com","password":"Smoke-Password-1!"}' + ACCESS_KEY="$(curl -fsS -b "$COOKIES" "$API/me" | jq -r '.accessKey // empty')" + [ -n "$ACCESS_KEY" ] || fail "no access key after login" + CONNECTION_ID="$(curl -fsS -b "$COOKIES" "$API/connections" | jq -r '.[0].id // empty')" + [ -n "$CONNECTION_ID" ] || fail "no default connection" + SOURCE_ID="$(curl -fsS -b "$COOKIES" -X POST "$API/sources" \ + -H 'Content-Type: application/json' \ + -d "{\"kind\":\"promql\",\"name\":\"PromQL\",\"connection\":\"$CONNECTION_ID\",\"from\":{\"databaseName\":\"default\",\"tableName\":\"metrics_ts\"},\"timestampValueExpression\":\"timestamp\"}" \ + | jq -r '.id // empty')" + [ -n "$SOURCE_ID" ] || fail "could not create a PromQL source" + + # The collector picks up INGESTION_API_KEY over OpAMP and batches its + # remote-write, so keep sending a fresh sample until one is in the table. + echo "sending OTLP gauge until it reaches default.metrics_ts..." + for i in $(seq 1 60); do + PAYLOAD="{\"resourceMetrics\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"ci-smoke\"}}]},\"scopeMetrics\":[{\"metrics\":[{\"name\":\"smoke_promql_gauge\",\"gauge\":{\"dataPoints\":[{\"timeUnixNano\":\"$(date +%s)000000000\",\"asDouble\":1,\"attributes\":[{\"key\":\"canary\",\"value\":{\"stringValue\":\"$CANARY\"}}]}]}}]}]}]}" + curl -fsS -X POST "http://localhost:$OTLP_PORT/v1/metrics" \ + -H 'Content-Type: application/json' -H "Authorization: $INGESTION_KEY" \ + -d "$PAYLOAD" >/dev/null 2>&1 || true + n="$(docker exec "$NAME" clickhouse-client --allow_experimental_time_series_table=1 --query \ + "SELECT count() FROM timeSeriesTags(default, metrics_ts) WHERE metric_name = 'smoke_promql_gauge' AND tags['canary'] = '$CANARY'" 2>/dev/null || echo 0)" + n="${n//[!0-9]/}" + [ "${n:-0}" -ge 1 ] && break + [ "$i" = 60 ] && fail "the gauge never reached default.metrics_ts" + sleep 2 + done + echo "gauge remote-written: collector -> clickhouse metrics_ts" + + # Prints a tool's result text; fails on an MCP or tool error. + mcp() { + local response text + response="$(curl -fsS "$API/mcp" \ + -H "Authorization: Bearer $ACCESS_KEY" \ + -H 'Content-Type: application/json' \ + -H 'Accept: application/json, text/event-stream' \ + -d "$(jq -nc --arg tool "$1" --argjson args "$2" \ + '{jsonrpc:"2.0",id:1,method:"tools/call",params:{name:$tool,arguments:$args}}')")" \ + || fail "$1: request failed" + # Streamable HTTP answers with one SSE event or a JSON body. + response="$(printf '%s\n' "$response" | sed -n 's/^data: //p' | grep . || printf '%s' "$response")" + text="$(jq -r '.result.content[0].text // .error.message' <<<"$response")" + [ "$(jq -r '.result.isError // false' <<<"$response")" = false ] || fail "$1: $text" + printf '%s' "$text" + } + + SELECTOR="smoke_promql_gauge{canary=\"$CANARY\"}" + rows="$(mcp clickstack_promql "$(jq -nc --arg s "$SOURCE_ID" --arg e "$SELECTOR" \ + '{sourceId:$s,expression:$e}')" | jq '.result.data | length')" + [ "${rows:-0}" -ge 1 ] || fail "clickstack_promql returned no rows" + echo "clickstack_promql returned $rows rows" + + DASHBOARD_ID="$(mcp clickstack_save_dashboard "$(jq -nc --arg s "$SOURCE_ID" --arg e "$SELECTOR" \ + '{name:"PromQL smoke",tiles:[{name:"Gauge",config:{configType:"promql",displayType:"line",sourceId:$s,expressions:[{expression:$e}]}}]}')" \ + | jq -r '.id')" + tile="$(mcp clickstack_query_tiles "$(jq -nc --arg d "$DASHBOARD_ID" '{dashboardId:$d}')" \ + | jq -c '.tiles[0]')" + [ "$(jq -r '.status == "ok" and .hasData == true' <<<"$tile")" = true ] \ + || fail "PromQL tile did not return data: $tile" + echo "promql-mcp smoke passed: OTLP -> metrics_ts -> clickstack_promql and a PromQL tile over MCP" + exit 0 +fi + CANARY="smoke-canary-$(date +%s)-$$" NOW_NS="$(date +%s)000000000" PAYLOAD="{\"resourceLogs\":[{\"resource\":{\"attributes\":[{\"key\":\"service.name\",\"value\":{\"stringValue\":\"ci-smoke\"}}]},\"scopeLogs\":[{\"logRecords\":[{\"timeUnixNano\":\"$NOW_NS\",\"severityText\":\"INFO\",\"body\":{\"stringValue\":\"$CANARY\"}}]}]}]}" diff --git a/scripts/slots.sh b/scripts/slots.sh index 62b2ea5ab9..65b185943b 100755 --- a/scripts/slots.sh +++ b/scripts/slots.sh @@ -70,16 +70,17 @@ hdx_dev_ports() { } # Integration tests (make dev-int): ClickHouse 18123+, MongoDB 39999+, -# API 19000+, OpAMP 14320+. +# API 19000+, Prometheus 19100+, OpAMP 14320+. hdx_ci_ports() { HDX_CI_SLOT="${HDX_CI_SLOT:-$(_hdx_default_slot "$1")}" HDX_CI_CH_PORT=$((18123 + HDX_CI_SLOT)) HDX_CI_MONGO_PORT=$((39999 + HDX_CI_SLOT)) HDX_CI_API_PORT=$((19000 + HDX_CI_SLOT)) HDX_CI_OPAMP_PORT=$((14320 + HDX_CI_SLOT)) + HDX_CI_PROMETHEUS_PORT=$((19100 + HDX_CI_SLOT)) HDX_CI_PROJECT="int-${HDX_CI_SLOT}" export HDX_CI_SLOT HDX_CI_CH_PORT HDX_CI_MONGO_PORT HDX_CI_API_PORT \ - HDX_CI_OPAMP_PORT HDX_CI_PROJECT + HDX_CI_OPAMP_PORT HDX_CI_PROMETHEUS_PORT HDX_CI_PROJECT } # E2E tests (make e2e): 20320-21399, plus the Playwright report server at