From d26dd42199d98bbb98b7b1727f9ae3b0630a88e2 Mon Sep 17 00:00:00 2001 From: nikkikapadia Date: Thu, 1 Oct 2026 15:36:58 -0400 Subject: [PATCH 1/3] test(search-events): evals for attribute fetching with context mocks --- .../mcp-core/src/api-client/client.test.ts | 33 +++ packages/mcp-core/src/api-client/client.ts | 7 + .../src/tools/support/search-events/utils.ts | 2 + ...rch-events-agent-attribute-context.eval.ts | 204 +++++++++++++ ...-attributes-spans-number-with-context.json | 57 ++++ ...-attributes-spans-string-with-context.json | 271 ++++++++++++++++++ .../trace-items-attributes-spans-string.json | 8 + packages/mcp-server-mocks/src/index.ts | 19 ++ packages/mcp-server-mocks/src/payloads.ts | 8 + 9 files changed, 609 insertions(+) create mode 100644 packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts create mode 100644 packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json create mode 100644 packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json diff --git a/packages/mcp-core/src/api-client/client.test.ts b/packages/mcp-core/src/api-client/client.test.ts index 869d04b9b..18be73abf 100644 --- a/packages/mcp-core/src/api-client/client.test.ts +++ b/packages/mcp-core/src/api-client/client.test.ts @@ -1978,6 +1978,39 @@ describe("API query builders", () => { expect(params.get("substringMatch")).toBe("tags["); expect(params.get("query")).toBe('transaction:"VPN connections"'); expect(params.get("attributeType")).toBeNull(); + expect(params.get("expand")).toBeNull(); + }); + + it("should request attribute context when context is enabled", async () => { + const apiService = new SentryApiService({ + host: "sentry.io", + accessToken: "test-token", + }); + const urls: string[] = []; + + globalThis.fetch = vi.fn().mockImplementation((url: string) => { + urls.push(url); + + return Promise.resolve({ + ok: true, + headers: { + get: (key: string) => + key === "content-type" ? "application/json" : null, + }, + json: () => Promise.resolve([]), + }); + }); + + await apiService.listTraceItemAttributes({ + organizationSlug: "test-org", + itemType: "spans", + context: true, + }); + + expect(urls).toHaveLength(1); + const params = new URL(urls[0]!).searchParams; + expect(params.get("itemType")).toBe("spans"); + expect(params.get("expand")).toBe("context"); }); it("should validate events requests via the validate endpoint", async () => { diff --git a/packages/mcp-core/src/api-client/client.ts b/packages/mcp-core/src/api-client/client.ts index 95325c0be..115a3687c 100644 --- a/packages/mcp-core/src/api-client/client.ts +++ b/packages/mcp-core/src/api-client/client.ts @@ -3698,6 +3698,7 @@ export class SentryApiService { attributeTypes, substringMatch, query, + context, }: { organizationSlug: string; itemType?: TraceItemType; @@ -3708,6 +3709,7 @@ export class SentryApiService { attributeTypes?: TraceItemAttributeType[]; substringMatch?: string; query?: string; + context?: boolean; }, opts?: RequestOptions, ): Promise { @@ -3720,6 +3722,7 @@ export class SentryApiService { end, substringMatch, query, + context, opts, ); @@ -3831,6 +3834,7 @@ export class SentryApiService { end?: string, substringMatch?: string, query?: string, + context?: boolean, opts?: RequestOptions, ): Promise { const queryParams = new URLSearchParams(); @@ -3844,6 +3848,9 @@ export class SentryApiService { if (query) { queryParams.set("query", query); } + if (context) { + queryParams.set("expand", "context"); + } this.applyTimeParams(queryParams, statsPeriod, start, end); const url = diff --git a/packages/mcp-core/src/tools/support/search-events/utils.ts b/packages/mcp-core/src/tools/support/search-events/utils.ts index 74ffd2b68..6641fdfe1 100644 --- a/packages/mcp-core/src/tools/support/search-events/utils.ts +++ b/packages/mcp-core/src/tools/support/search-events/utils.ts @@ -590,6 +590,7 @@ export async function fetchCustomAttributes( attributeTypes?: TraceItemAttributeType[]; substringMatch?: string; query?: string; + context?: boolean; } = {}, ): Promise<{ attributes: Record; @@ -640,6 +641,7 @@ export async function fetchCustomAttributes( attributeTypes: options.attributeTypes, substringMatch: options.substringMatch, query: options.query, + context: options.context, }); for (const attr of attributesResponse) { diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts new file mode 100644 index 000000000..453c8864a --- /dev/null +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -0,0 +1,204 @@ +import { describeEval } from "vitest-evals"; +import { ToolCallScorer } from "vitest-evals"; +import { searchEventsAgent } from "@sentry/mcp-core/tools/search-events/agent"; +import { SentryApiService } from "@sentry/mcp-core/api-client"; +import { StructuredOutputScorer } from "./utils/structuredOutputScorer"; +import "../setup-env"; + +// Compares the attribute the agent picks when the trace-items attributes +// endpoint is called with and without `expand=context`. The with-context span +// fixtures list the replacement for each deprecated attribute (e.g. +// user.geo.region for geo.region), so only that path should use it. +// +// The datasetAttributes tool does not request context itself, so this service +// forces it on or off for every attributes request. +class AttributeContextApiService extends SentryApiService { + constructor(private readonly attributeContext: boolean) { + super({ accessToken: "test-token" }); + } + + override listTraceItemAttributes( + ...[params, opts]: Parameters + ) { + return super.listTraceItemAttributes( + { ...params, context: this.attributeContext }, + opts, + ); + } +} + +describeEval("search-events-agent-attributes-without-context", { + data: async () => { + return [ + { + // Without context the agent picks the deprecated geo.region + input: + "Show me spans from users in the Ontario region in the last 24 hours", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + /(^|\s)geo\.region:/.test(value) && + !value.includes("user.geo.region"), + }, + }, + { + // http.method is also picked from the static span fields + input: "Count spans grouped by HTTP method over the last 7 days", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + value.includes("http.method") && + !value.includes("http.request.method"), + }, + }, + { + // http.status_code is also picked from static span fields + input: "Show me spans with HTTP status code 503 in the last 24 hours", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("http.status_code:503") && + !value.includes("http.response.status_code"), + }, + }, + ]; + }, + task: async (input) => { + // Create an API service that never requests attribute context + const apiService = new AttributeContextApiService(false); + + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); + + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }, + scorers: [ + ToolCallScorer({ params: "fuzzy" }), // Validates tool calls + StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], +}); + +// Context not enabled yet so these should not show the correct queries. Asserting them to the wrong queries for now. +describeEval("search-events-agent-attributes-with-context", { + data: async () => { + return [ + { + // EVENTYALLY user.geo.region is chosen due to deprecation + input: + "Show me spans from users in the Ontario region in the last 24 hours", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + value.includes("user.geo.region:") && + !/(^|\s)geo\.region:/.test(value), + }, + }, + { + // EVENTUALLY Context marks http.method as deprecated in favor of http.request.method + input: "Count spans grouped by HTTP method over the last 7 days", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + fields: (value: unknown) => + Array.isArray(value) && + !value.includes("http.request.method") && + value.includes("http.method"), + }, + }, + { + // EVENTUALLY Context marks http.status_code as deprecated in favor of + // http.response.status_code + input: "Show me spans with HTTP status code 503 in the last 24 hours", + expectedTools: [ + { + name: "datasetAttributes", + arguments: { + dataset: "spans", + }, + }, + ], + expected: { + dataset: "spans", + query: (value: unknown) => + typeof value === "string" && + !value.includes("http.response.status_code:503") && + value.includes("http.status_code"), + }, + }, + ]; + }, + task: async (input) => { + // Create an API service that always requests attribute context + const apiService = new AttributeContextApiService(true); + + const agentResult = await searchEventsAgent({ + query: input, + organizationSlug: "sentry-mcp-evals", + apiService, + }); + + return { + result: JSON.stringify(agentResult.result), + toolCalls: agentResult.toolCalls.map((call: any) => ({ + name: call.toolName, + arguments: call.args, + })), + }; + }, + scorers: [ + ToolCallScorer({ params: "fuzzy" }), // Validates tool calls + StructuredOutputScorer({ match: "fuzzy" }), // Validates the structured query output with flexible matching + ], +}); diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json new file mode 100644 index 000000000..e53191dcf --- /dev/null +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-number-with-context.json @@ -0,0 +1,57 @@ +[ + { + "key": "span.duration", + "name": "span.duration", + "context": { + "isConvention": false, + "brief": "The total time taken by the span.", + "isDeprecated": false + } + }, + { + "key": "transaction.duration", + "name": "transaction.duration", + "context": {} + }, + { + "key": "http.status_code", + "name": "HTTP Status Code", + "context": { + "isConvention": true, + "brief": "The status code of the HTTP response.", + "isDeprecated": true, + "examples": [404], + "replacementAttribute": "http.response.status_code" + } + }, + { + "key": "http.response.status_code", + "name": "http.response.status_code", + "context": { + "isConvention": true, + "brief": "The status code of the HTTP response.", + "isDeprecated": false, + "examples": [404] + } + }, + { + "key": "custom.count", + "name": "custom.count", + "context": {} + }, + { + "key": "custom.score", + "name": "custom.score", + "context": {} + }, + { + "key": "custom.latency_ms", + "name": "custom.latency_ms", + "context": {} + }, + { + "key": "custom.db.pool_size", + "name": "custom.db.pool_size", + "context": {} + } +] diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json new file mode 100644 index 000000000..ed5e763d6 --- /dev/null +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string-with-context.json @@ -0,0 +1,271 @@ +[ + { + "key": "span.op", + "name": "span.op", + "context": { + "isConvention": true, + "brief": "The operation of a span.", + "isDeprecated": false, + "examples": [ + "http.client" + ] + } + }, + { + "key": "span.description", + "name": "span.description", + "secondaryAliases": [ + "description", + "message" + ], + "context": { + "isConvention": false, + "brief": "Description of the span's operation.", + "isDeprecated": false + } + }, + { + "key": "span.status", + "name": "span.status", + "context": { + "isConvention": true, + "brief": "The span's status (either \"ok\" or \"error\"). Older SDKs may set this to a more specific error, but this behaviour is deprecated.", + "isDeprecated": false, + "examples": [ + "ok" + ] + } + }, + { + "key": "transaction", + "name": "transaction", + "context": { + "isConvention": true, + "brief": "The sentry transaction (segment name).", + "isDeprecated": true, + "examples": [ + "GET /" + ], + "replacementAttribute": "sentry.segment.name" + } + }, + { + "key": "transaction.op", + "name": "transaction.op", + "context": { + "isConvention": false, + "brief": "Operation of the containing transaction.", + "isDeprecated": false + } + }, + { + "key": "transaction.status", + "name": "transaction.status", + "context": {} + }, + { + "key": "project", + "name": "project", + "context": { + "isConvention": false, + "brief": "The name of the project. In some pages of sentry.io, you can also filter on project using a dropdown.", + "isDeprecated": false + } + }, + { + "key": "environment", + "name": "environment", + "context": { + "isConvention": true, + "brief": "The sentry environment.", + "isDeprecated": true, + "examples": [ + "production" + ], + "replacementAttribute": "sentry.environment" + } + }, + { + "key": "release", + "name": "release", + "context": { + "isConvention": true, + "brief": "The sentry release.", + "isDeprecated": true, + "examples": [ + "production" + ], + "replacementAttribute": "sentry.release" + } + }, + { + "key": "user.id", + "name": "user.id", + "context": {} + }, + { + "key": "user.email", + "name": "user.email", + "context": {} + }, + { + "key": "user.username", + "name": "user.username", + "context": {} + }, + { + "key": "platform", + "name": "platform", + "context": { + "isConvention": true, + "brief": "The sdk platform that generated the event.", + "isDeprecated": false, + "examples": [ + "php" + ] + } + }, + { + "key": "sdk.name", + "name": "sdk.name", + "context": { + "isConvention": true, + "brief": "The sentry sdk name.", + "isDeprecated": false, + "examples": [ + "@sentry/react" + ] + } + }, + { + "key": "sdk.version", + "name": "sdk.version", + "context": { + "isConvention": true, + "brief": "The sentry sdk version.", + "isDeprecated": false, + "examples": [ + "7.0.0" + ] + } + }, + { + "key": "http.method", + "name": "http.method", + "context": { + "isConvention": true, + "brief": "The HTTP method used.", + "isDeprecated": true, + "examples": [ + "GET" + ], + "replacementAttribute": "http.request.method" + } + }, + { + "key": "http.request.method", + "name": "http.request.method", + "context": { + "isConvention": true, + "brief": "The HTTP method used.", + "isDeprecated": false, + "examples": [ + "GET" + ] + } + }, + { + "key": "http.url", + "name": "http.url", + "context": { + "isConvention": true, + "brief": "The URL of the resource that was fetched.", + "isDeprecated": true, + "examples": [ + "https://example.com/test?foo=bar#buzz" + ], + "replacementAttribute": "url.full" + } + }, + { + "key": "browser.name", + "name": "browser.name", + "context": {} + }, + { + "key": "os.name", + "name": "os.name", + "context": { + "isConvention": true, + "brief": "Human readable operating system name.", + "isDeprecated": false, + "examples": [ + "Ubuntu" + ] + } + }, + { + "key": "device", + "name": "device", + "context": {} + }, + { + "key": "geo.country_code", + "name": "geo.country_code", + "context": { + "isConvention": true, + "brief": "The country code of the user's location.", + "isDeprecated": false, + "examples": [ + "US" + ] + } + }, + { + "key": "geo.region", + "name": "geo.region", + "context": { + "isConvention": true, + "brief": "The region of the user's location.", + "isDeprecated": true, + "examples": [ + "Canada" + ], + "replacementAttribute": "user.geo.region" + } + }, + { + "key": "user.geo.region", + "name": "user.geo.region", + "context": { + "isConvention": true, + "brief": "The region of the user's location.", + "isDeprecated": false, + "examples": [ + "Canada" + ] + } + }, + { + "key": "geo.city", + "name": "geo.city", + "context": { + "isConvention": true, + "brief": "The city of the user's location.", + "isDeprecated": false, + "examples": [ + "Toronto" + ] + } + }, + { + "key": "custom.tier", + "name": "custom.tier", + "context": {} + }, + { + "key": "custom.feature_flag", + "name": "custom.feature_flag", + "context": {} + } +] diff --git a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json index 9fe1ec60c..115773a13 100644 --- a/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json +++ b/packages/mcp-server-mocks/src/fixtures/trace-items-attributes-spans-string.json @@ -63,6 +63,10 @@ "key": "http.method", "name": "HTTP Method" }, + { + "key": "http.request.method", + "name": "http.request.method" + }, { "key": "http.url", "name": "HTTP URL" @@ -87,6 +91,10 @@ "key": "geo.region", "name": "Geographic Region" }, + { + "key": "user.geo.region", + "name": "user.geo.region" + }, { "key": "geo.city", "name": "City" diff --git a/packages/mcp-server-mocks/src/index.ts b/packages/mcp-server-mocks/src/index.ts index 38e4929d4..b3376a3c2 100644 --- a/packages/mcp-server-mocks/src/index.ts +++ b/packages/mcp-server-mocks/src/index.ts @@ -129,6 +129,12 @@ import traceItemsAttributesSpansNumberFixture from "./fixtures/trace-items-attri import traceItemsAttributesSpansStringFixture from "./fixtures/trace-items-attributes-spans-string.json" with { type: "json", }; +import traceItemsAttributesSpansNumberWithContextFixture from "./fixtures/trace-items-attributes-spans-number-with-context.json" with { + type: "json", +}; +import traceItemsAttributesSpansStringWithContextFixture from "./fixtures/trace-items-attributes-spans-string-with-context.json" with { + type: "json", +}; import traceItemsAttributesTraceMetricsNumberFixture from "./fixtures/trace-items-attributes-tracemetrics-number.json" with { type: "json", }; @@ -1201,6 +1207,7 @@ export const restHandlers = buildHandlers([ const url = new URL(request.url); const itemType = url.searchParams.get("itemType"); const attributeType = url.searchParams.get("attributeType"); + const useContext = url.searchParams.get("expand") === "context"; // Validate required parameters if (!itemType) { @@ -1223,6 +1230,18 @@ export const restHandlers = buildHandlers([ if (!attributeType) { if (normalizedItemType === "span") { + if (useContext) { + return HttpResponse.json([ + ...withTraceItemAttributeMetadata( + traceItemsAttributesSpansStringWithContextFixture, + "string", + ), + ...withTraceItemAttributeMetadata( + traceItemsAttributesSpansNumberWithContextFixture, + "number", + ), + ]); + } return HttpResponse.json([ ...withTraceItemAttributeMetadata( traceItemsAttributesSpansStringFixture, diff --git a/packages/mcp-server-mocks/src/payloads.ts b/packages/mcp-server-mocks/src/payloads.ts index 606cc857b..5078d0fda 100644 --- a/packages/mcp-server-mocks/src/payloads.ts +++ b/packages/mcp-server-mocks/src/payloads.ts @@ -76,6 +76,12 @@ import traceItemsAttributesSpansNumberFixture from "./fixtures/trace-items-attri import traceItemsAttributesSpansStringFixture from "./fixtures/trace-items-attributes-spans-string.json" with { type: "json", }; +import traceItemsAttributesSpansNumberWithContextFixture from "./fixtures/trace-items-attributes-spans-number-with-context.json" with { + type: "json", +}; +import traceItemsAttributesSpansStringWithContextFixture from "./fixtures/trace-items-attributes-spans-string-with-context.json" with { + type: "json", +}; import traceItemsAttributesFixture from "./fixtures/trace-items-attributes.json" with { type: "json", }; @@ -129,6 +135,8 @@ export { traceItemsAttributesFixture, traceItemsAttributesSpansStringFixture, traceItemsAttributesSpansNumberFixture, + traceItemsAttributesSpansNumberWithContextFixture, + traceItemsAttributesSpansStringWithContextFixture, traceItemsAttributesLogsStringFixture, traceItemsAttributesLogsNumberFixture, traceMetaFixture, From 99d90f78078c7727961c03cb9380ce15c1a01dca Mon Sep 17 00:00:00 2001 From: nikkikapadia Date: Fri, 2 Oct 2026 10:24:58 -0400 Subject: [PATCH 2/3] fix typo in context eval --- .../src/evals/search-events-agent-attribute-context.eval.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 453c8864a..812916a71 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -134,8 +134,8 @@ describeEval("search-events-agent-attributes-with-context", { dataset: "spans", query: (value: unknown) => typeof value === "string" && - value.includes("user.geo.region:") && - !/(^|\s)geo\.region:/.test(value), + !value.includes("user.geo.region:") && + /(^|\s)geo\.region:/.test(value), }, }, { From 31073cb1d7fc13cdc9f8219088a13d9cad466af4 Mon Sep 17 00:00:00 2001 From: nikkikapadia Date: Fri, 2 Oct 2026 15:12:47 -0400 Subject: [PATCH 3/3] remove geo.region eval test --- ...rch-events-agent-attribute-context.eval.ts | 40 ------------------- 1 file changed, 40 deletions(-) diff --git a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts index 812916a71..85ae9fbff 100644 --- a/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts +++ b/packages/mcp-server-evals/src/evals/search-events-agent-attribute-context.eval.ts @@ -30,26 +30,6 @@ class AttributeContextApiService extends SentryApiService { describeEval("search-events-agent-attributes-without-context", { data: async () => { return [ - { - // Without context the agent picks the deprecated geo.region - input: - "Show me spans from users in the Ontario region in the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { - dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - /(^|\s)geo\.region:/.test(value) && - !value.includes("user.geo.region"), - }, - }, { // http.method is also picked from the static span fields input: "Count spans grouped by HTTP method over the last 7 days", @@ -118,26 +98,6 @@ describeEval("search-events-agent-attributes-without-context", { describeEval("search-events-agent-attributes-with-context", { data: async () => { return [ - { - // EVENTYALLY user.geo.region is chosen due to deprecation - input: - "Show me spans from users in the Ontario region in the last 24 hours", - expectedTools: [ - { - name: "datasetAttributes", - arguments: { - dataset: "spans", - }, - }, - ], - expected: { - dataset: "spans", - query: (value: unknown) => - typeof value === "string" && - !value.includes("user.geo.region:") && - /(^|\s)geo\.region:/.test(value), - }, - }, { // EVENTUALLY Context marks http.method as deprecated in favor of http.request.method input: "Count spans grouped by HTTP method over the last 7 days",