From 2cebcae462e9d3437d979fbedbd00b91a0704f39 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Mon, 28 Sep 2026 12:52:25 +0530 Subject: [PATCH 1/7] fix(loadtesting): back every capability's returns with a typed response schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every capability resolved its 2xx to the shared SuccessEnvelope, whose data is a typeless object — so returns carried the whole payload contract as a flat list of names with no types, describeCapability returned an empty schema for all 21, and the check-contract drift guard could never catch a dropped field. Add a per- capability Data schema and compose each 2xx as allOf[SuccessEnvelope, {data}], so describeCapability exposes real field types and drift is detectable. scripts/check-contract.py now reports 0 unbacked for loadtesting (was 21/75). Also, derivable contract fixes: - one-line description on all 9 entities (the field listProducts routes on) - format date-time on the ISO timestamp params + uuid on compare run ids - minimum/maximum on the limit params from their declared caps - examples on high-leverage create/search/compare params - updateLoadTest guidance: ifVersion opt-in vs last-write-wins semantics - searchLoadTests guidance: hasMore has no cursor (top-N, narrow the query) Tests updated: the two assertions that documented the missing entity descriptions now assert they are present and meet the shared quality bar. --- capability/loadtesting.capability-index.json | 743 ++++++++++++++++-- scripts/contract-baseline/loadtesting.json | 116 +-- tests/tools/capabilityRegistryE2E.test.ts | 7 +- tests/tools/capabilityRegistryRouting.test.ts | 35 +- 4 files changed, 719 insertions(+), 182 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index f31dc4b2..818e8bcc 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -28,7 +28,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestPlatformStatusData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -51,7 +64,9 @@ { "name": "limit", "type": "integer", - "description": "Max items to return (1-100, default 50)." + "description": "Max items to return (1-100, default 50).", + "minimum": 1, + "maximum": 100 }, { "name": "cursor", @@ -85,7 +100,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "ListLoadTestProjectsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -117,7 +145,9 @@ { "name": "limit", "type": "integer", - "description": "Max items to return (1-100, default 50)." + "description": "Max items to return (1-100, default 50).", + "minimum": 1, + "maximum": 100 }, { "name": "cursor", @@ -159,7 +189,8 @@ "name": "updatedSince", "type": "string", "description": "ISO-8601 timestamp; only tests updated after it.", - "example": "2026-08-01T00:00:00Z" + "example": "2026-08-01T00:00:00Z", + "format": "date-time" } ], "intent": "List the load tests in a project — use this to find a testId or browse tests, optionally filtered by type, framework, tag or recent activity.", @@ -177,7 +208,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "ListLoadTestsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -205,19 +249,23 @@ { "name": "name", "type": "string", - "description": "Test name to search for (substring match, case-insensitive); required." + "description": "Test name to search for (substring match, case-insensitive); required.", + "example": "checkout" }, { "name": "limit", "type": "integer", - "description": "Max items to return (1-100, default 50)." + "description": "Max items to return (1-100, default 50).", + "minimum": 1, + "maximum": 100 } ], "intent": "Find a load test by name across ALL of the caller's projects — use this to resolve a named test to a testId without knowing which project it is in.", "guidance": [ "Group-scoped: unlike listLoadTests it needs no projectId. Prefer it to resolve a named test in one call instead of paging every project.", "Each row carries testId, projectId, name, testType and framework — disambiguate same-named tests in different projects by projectId.", - "Returns one row per test (its latest version); the numeric testId is what getLoadTest and startLoadTestRun need." + "Returns one row per test (its latest version); the numeric testId is what getLoadTest and startLoadTestRun need.", + "hasMore:true means more matches exist beyond the returned limit — there is NO cursor and no page 2; narrow the name query or raise limit to surface them (results are the top matches, not a paginated list)." ], "returns": [ "tests", @@ -225,7 +273,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "SearchLoadTestsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -284,12 +345,14 @@ "name": "sinceIso", "type": "string", "description": "ISO-8601 lower bound.", - "example": "2026-08-01T00:00:00Z" + "example": "2026-08-01T00:00:00Z", + "format": "date-time" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound." + "description": "ISO-8601 upper bound.", + "format": "date-time" } ], "intent": "Show how a project's load-test metrics trend across recent runs — use this for 'is performance getting better or worse across this project?'", @@ -303,7 +366,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestProjectTrendsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -336,7 +412,9 @@ { "name": "limit", "type": "integer", - "description": "Max items to return (1-100, default 50)." + "description": "Max items to return (1-100, default 50).", + "minimum": 1, + "maximum": 100 }, { "name": "cursor", @@ -372,12 +450,14 @@ "name": "sinceIso", "type": "string", "description": "ISO-8601 lower bound.", - "example": "2026-08-01T00:00:00Z" + "example": "2026-08-01T00:00:00Z", + "format": "date-time" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound." + "description": "ISO-8601 upper bound.", + "format": "date-time" }, { "name": "hadSlaBreach", @@ -405,7 +485,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "ListProjectLoadTestRunsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -433,7 +526,9 @@ { "name": "limit", "type": "integer", - "description": "Max items to return (1-200, default 50)." + "description": "Max items to return (1-200, default 50).", + "minimum": 1, + "maximum": 200 }, { "name": "minElapsedSec", @@ -468,7 +563,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "ListActiveLoadTestRunsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -519,7 +627,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestRunStatusData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -575,7 +696,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestInsightSummaryData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -667,12 +801,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound; slices the report to data in this time window." + "description": "ISO-8601 lower bound; slices the report to data in this time window.", + "format": "date-time" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound for the time-window slice." + "description": "ISO-8601 upper bound for the time-window slice.", + "format": "date-time" } ], "intent": "Get the full metrics report for a completed run — KPIs, SLA verdicts, per-transaction and error breakdowns. Use this to answer 'how did this run perform?'", @@ -699,7 +835,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestRunReportData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -741,7 +890,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "StopLoadTestRunData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -775,7 +937,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestQuotaData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -799,7 +974,8 @@ "name": "testId", "type": "integer", "required": true, - "description": "The test to price." + "description": "The test to price.", + "example": 1234 }, { "name": "overrides", @@ -836,7 +1012,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "EstimateLoadTestRunCostData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -860,13 +1049,17 @@ "name": "baselineRunId", "type": "string", "required": true, - "description": "Run UUID to compare against." + "description": "Run UUID to compare against.", + "format": "uuid", + "example": "6f1d2c9a-7b3e-4a10-9c22-0e5f8a1b2c3d" }, { "name": "candidateRunId", "type": "string", "required": true, - "description": "Run UUID being evaluated; must differ from baselineRunId." + "description": "Run UUID being evaluated; must differ from baselineRunId.", + "format": "uuid", + "example": "b2e4f6a8-1c3d-4e5f-8a90-2b4c6d8e0f11" }, { "name": "dimensions", @@ -916,7 +1109,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "CompareLoadTestRunsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -941,7 +1147,8 @@ "type": "string", "required": true, "maxLength": 255, - "description": "Test name. Max 255 characters — truncate before sending; a longer name is rejected with a 400." + "description": "Test name. Max 255 characters — truncate before sending; a longer name is rejected with a 400.", + "example": "checkout-baseline" }, { "name": "testType", @@ -1043,7 +1250,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "CreateLoadTestData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1125,7 +1345,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "StartLoadTestRunData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1158,7 +1391,9 @@ { "name": "limit", "type": "integer", - "description": "Max items to return (1-100, default 50)." + "description": "Max items to return (1-100, default 50).", + "minimum": 1, + "maximum": 100 }, { "name": "cursor", @@ -1178,12 +1413,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound." + "description": "ISO-8601 lower bound.", + "format": "date-time" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound." + "description": "ISO-8601 upper bound.", + "format": "date-time" }, { "name": "verdict", @@ -1222,7 +1459,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "ListLoadTestRunsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1285,12 +1535,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound." + "description": "ISO-8601 lower bound.", + "format": "date-time" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound." + "description": "ISO-8601 upper bound.", + "format": "date-time" } ], "intent": "Show how one test's metrics trend across its recent runs — use this for 'is this test getting slower over time?'", @@ -1304,7 +1556,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestHistoricalTrendsData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1345,7 +1610,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestMetricsManifestData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1388,7 +1666,8 @@ { "name": "ifVersion", "type": "string", - "description": "Optimistic-concurrency guard (ISO timestamp of the version being edited)." + "description": "Optimistic-concurrency guard (ISO timestamp of the version being edited).", + "format": "date-time" }, { "name": "validationToken", @@ -1424,7 +1703,8 @@ "config is a partial update, but each field it carries REPLACES that field wholesale — it does not merge. tags overwrites the entire tag set; it does not append. To add a tag to a test (or the same tag across several tests), getLoadTest each one first and send the union under config.tags.", "To change the load profile, send config.loadProfile plus that profile's fields (see createLoadTest's load-profile guidance for the four shapes). Switching profiles clears the previous profile's fields — e.g. ramping→throughput empties the stages and drops iterations. iterations and throughput are PLU-only.", "To tag a specific RUN rather than the test, pass runId (a run UUID from listLoadTestRuns) together with config.tags — the tags apply to that one run and the saved test is untouched. A run inherits the test's tags until you set its own; setting run tags overrides (they also replace wholesale, so send the union to add). Omit runId to tag the test itself.", - "Tags live on the load test, not on its runs — there is no per-run tagging, so do not touch runs when asked to tag a test. To tag every test in a project, list them with listLoadTests and page through with cursor until hasMore is false, then updateLoadTest each one — do not stop after the first page or a subset." + "Tags live on the load test, not on its runs — there is no per-run tagging, so do not touch runs when asked to tag a test. To tag every test in a project, list them with listLoadTests and page through with cursor until hasMore is false, then updateLoadTest each one — do not stop after the first page or a subset.", + "Optimistic concurrency is opt-in: pass ifVersion (the version token from a prior get/create/update) so the edit fails with 409 CONFLICT if the test changed meanwhile; OMITTING ifVersion is a last-write-wins update that overwrites any concurrent change." ], "returns": [ "testId", @@ -1432,7 +1712,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "UpdateLoadTestData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1494,7 +1787,20 @@ ], "responses": { "200": { - "$response": "Success" + "schema": { + "allOf": [ + { + "$schema": "SuccessEnvelope" + }, + { + "properties": { + "data": { + "$schema": "GetLoadTestData" + } + } + } + ] + } }, "400": { "$response": "BadRequest" @@ -1549,6 +1855,342 @@ "success", "error" ] + }, + "GetLoadTestPlatformStatusData": { + "type": "object", + "properties": { + "platformState": { + "type": "string" + }, + "components": { + "type": "array" + }, + "checkedAt": { + "type": "string", + "format": "date-time" + } + } + }, + "ListLoadTestProjectsData": { + "type": "object", + "properties": { + "projects": { + "type": "array" + }, + "hasMore": { + "type": "boolean" + }, + "nextCursor": { + "type": "string" + } + } + }, + "ListLoadTestsData": { + "type": "object", + "properties": { + "tests": { + "type": "array" + }, + "hasMore": { + "type": "boolean" + }, + "nextCursor": { + "type": "string" + } + } + }, + "SearchLoadTestsData": { + "type": "object", + "properties": { + "tests": { + "type": "array" + }, + "hasMore": { + "type": "boolean" + } + } + }, + "GetLoadTestProjectTrendsData": { + "type": "object", + "properties": { + "metrics": { + "description": "Trend series payload; shape per getLoadTestMetricsManifest." + } + } + }, + "ListProjectLoadTestRunsData": { + "type": "object", + "properties": { + "runs": { + "type": "array" + }, + "hasMore": { + "type": "boolean" + }, + "nextCursor": { + "type": "string" + } + } + }, + "ListActiveLoadTestRunsData": { + "type": "object", + "properties": { + "runs": { + "type": "array" + } + } + }, + "GetLoadTestRunStatusData": { + "type": "object", + "properties": { + "status": { + "type": "string" + }, + "elapsedSec": { + "type": "number" + }, + "remainingSec": { + "type": "number" + }, + "pollAfterSeconds": { + "type": "number" + }, + "slaBreachFlags": { + "type": "array" + } + } + }, + "GetLoadTestInsightSummaryData": { + "type": "object", + "properties": { + "state": { + "type": "string" + }, + "status": { + "type": "string" + }, + "report": { + "type": "object" + } + } + }, + "GetLoadTestRunReportData": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "format": "uuid" + }, + "testType": { + "type": "string" + }, + "status": { + "type": "string" + }, + "testName": { + "type": "string" + }, + "durationSec": { + "type": "number" + }, + "kpis": { + "type": "object" + }, + "slaVerdicts": { + "type": "array" + }, + "transactions": { + "type": "array" + }, + "errorsByCategory": { + "type": "object" + }, + "meta": { + "type": "object" + } + } + }, + "StopLoadTestRunData": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "format": "uuid" + }, + "status": { + "type": "string" + }, + "stoppedAt": { + "type": "string", + "format": "date-time" + } + } + }, + "GetLoadTestQuotaData": { + "type": "object", + "properties": { + "plan": { + "type": "object" + }, + "vuHours": { + "type": "object" + }, + "concurrency": { + "type": "object" + } + } + }, + "EstimateLoadTestRunCostData": { + "type": "object", + "properties": { + "estimatedVuHours": { + "type": "number" + }, + "estimationBasis": { + "type": "string" + }, + "rangeBasis": { + "type": "string" + }, + "rangeExplanation": { + "type": "string" + }, + "fitsInQuota": { + "type": "boolean" + }, + "remainingAfterEstimate": { + "type": "number" + } + } + }, + "CompareLoadTestRunsData": { + "type": "object", + "properties": { + "kpiDeltas": { + "description": "Per-KPI baseline→candidate deltas." + }, + "transactionDeltas": { + "type": "array" + } + } + }, + "CreateLoadTestData": { + "type": "object", + "properties": { + "testId": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "projectId": { + "type": "integer" + }, + "version": { + "type": "string" + }, + "dashboardLink": { + "type": "string" + } + } + }, + "StartLoadTestRunData": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "format": "uuid" + }, + "status": { + "type": "string" + }, + "startedAt": { + "type": "string", + "format": "date-time" + }, + "dashboardLink": { + "type": "string" + }, + "estimatedVuHoursRange": { + "description": "Low/high VU-hour estimate for the started run." + } + } + }, + "ListLoadTestRunsData": { + "type": "object", + "properties": { + "runs": { + "type": "array" + }, + "hasMore": { + "type": "boolean" + }, + "nextCursor": { + "type": "string" + } + } + }, + "GetLoadTestHistoricalTrendsData": { + "type": "object", + "properties": { + "metrics": { + "description": "Trend series payload; shape per getLoadTestMetricsManifest." + } + } + }, + "GetLoadTestMetricsManifestData": { + "type": "object", + "properties": { + "testType": { + "type": "string" + }, + "metrics": { + "type": "array" + }, + "aliases": { + "type": "object" + } + } + }, + "UpdateLoadTestData": { + "type": "object", + "properties": { + "testId": { + "type": "integer" + }, + "version": { + "type": "string" + } + } + }, + "GetLoadTestData": { + "type": "object", + "properties": { + "testId": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "testType": { + "type": "string" + }, + "framework": { + "type": "string" + }, + "version": { + "type": "string" + }, + "projectId": { + "type": "integer" + }, + "ownerUserId": { + "type": "integer" + }, + "config": { + "type": "object" + } + } } }, "responses": { @@ -1613,6 +2255,7 @@ "loadTest": { "entity": "loadTest", "title": "Load Test", + "description": "A saved load-test configuration (framework, script, load profile and thresholds); addressed by a numeric testId. Runs are executions of it.", "aliases": [ "test", "load test", @@ -1652,6 +2295,7 @@ "run": { "entity": "run", "title": "Run", + "description": "A single execution of a load test; addressed by runId (a UUID, also called jobId). Carries status, metrics and a report.", "aliases": [ "execution", "job", @@ -1690,6 +2334,7 @@ "project": { "entity": "project", "title": "Project", + "description": "A container that groups related load tests under a group/account; addressed by a numeric projectId.", "aliases": [ "project", "workspace", @@ -1715,6 +2360,7 @@ "report": { "entity": "report", "title": "Run Report", + "description": "The results of a completed run (KPIs, SLA verdicts, per-transaction rows, error breakdown); addressed by the run's runId.", "aliases": [ "report", "result", @@ -1742,6 +2388,7 @@ "insight": { "entity": "insight", "title": "AI Insight", + "description": "AI-generated analysis of a completed run (root cause and recommendations); addressed by the run's runId, and may be pending until generated.", "aliases": [ "insight", "ai insight", @@ -1768,6 +2415,7 @@ "trend": { "entity": "trend", "title": "Historical Trend", + "description": "Historical time-series of a metric across a test's or project's runs; scoped by testId or projectId, not independently identified.", "aliases": [ "trend", "trends", @@ -1800,6 +2448,7 @@ "quota": { "entity": "quota", "title": "Quota & Cost", + "description": "The caller's VU-hour entitlement, usage and concurrency limits; account/group-scoped, with no id.", "aliases": [ "quota", "vu hours", @@ -1829,6 +2478,7 @@ "metrics": { "entity": "metrics", "title": "Metrics Manifest", + "description": "The per-testType catalogue of available metric names, @-aliases and groupBy dimensions; a manifest, not an instance.", "aliases": [ "metric", "metrics", @@ -1856,6 +2506,7 @@ "platform": { "entity": "platform", "title": "Platform Status", + "description": "Health status of the Load Testing service and its dependencies; a singleton probe with no id.", "aliases": [ "platform", "status", diff --git a/scripts/contract-baseline/loadtesting.json b/scripts/contract-baseline/loadtesting.json index eda02fa3..955b306f 100644 --- a/scripts/contract-baseline/loadtesting.json +++ b/scripts/contract-baseline/loadtesting.json @@ -2,119 +2,5 @@ "empty_2xx": [], "index": "capability/loadtesting.capability-index.json", "no_2xx": [], - "unbacked": { - "compareLoadTestRuns": [ - "kpiDeltas", - "transactionDeltas" - ], - "createLoadTest": [ - "dashboardLink", - "name", - "projectId", - "testId", - "version" - ], - "estimateLoadTestRunCost": [ - "estimatedVuHours", - "estimationBasis", - "fitsInQuota", - "rangeBasis", - "rangeExplanation", - "remainingAfterEstimate" - ], - "getLoadTest": [ - "config", - "framework", - "name", - "ownerUserId", - "projectId", - "testId", - "testType", - "version" - ], - "getLoadTestHistoricalTrends": [ - "metrics" - ], - "getLoadTestInsightSummary": [ - "report", - "state", - "status" - ], - "getLoadTestMetricsManifest": [ - "aliases", - "metrics", - "testType" - ], - "getLoadTestPlatformStatus": [ - "checkedAt", - "components", - "platformState" - ], - "getLoadTestProjectTrends": [ - "metrics" - ], - "getLoadTestQuota": [ - "concurrency", - "plan", - "vuHours" - ], - "getLoadTestRunReport": [ - "durationSec", - "errorsByCategory", - "kpis", - "meta", - "runId", - "slaVerdicts", - "status", - "testName", - "testType", - "transactions" - ], - "getLoadTestRunStatus": [ - "elapsedSec", - "pollAfterSeconds", - "remainingSec", - "slaBreachFlags", - "status" - ], - "listActiveLoadTestRuns": [ - "runs" - ], - "listLoadTestProjects": [ - "hasMore", - "nextCursor", - "projects" - ], - "listLoadTestRuns": [ - "hasMore", - "nextCursor", - "runs" - ], - "listLoadTests": [ - "hasMore", - "nextCursor", - "tests" - ], - "listProjectLoadTestRuns": [ - "hasMore", - "nextCursor", - "runs" - ], - "startLoadTestRun": [ - "dashboardLink", - "estimatedVuHoursRange", - "runId", - "startedAt", - "status" - ], - "stopLoadTestRun": [ - "runId", - "status", - "stoppedAt" - ], - "updateLoadTest": [ - "testId", - "version" - ] - } + "unbacked": {} } diff --git a/tests/tools/capabilityRegistryE2E.test.ts b/tests/tools/capabilityRegistryE2E.test.ts index 3177df98..c3b22e86 100644 --- a/tests/tools/capabilityRegistryE2E.test.ts +++ b/tests/tools/capabilityRegistryE2E.test.ts @@ -255,13 +255,12 @@ describe("capability registry, end to end through the server factory", () => { expect(tm.shared_terms[0]).toMatchObject({ term: "project", entity: "project" }); expect(tm.shared_terms[0].means).toMatch(/top-level container/); - // Load Testing ships no entity descriptions, so its sense of `project` has no - // `means`. Absent rather than invented: the question is still askable, just thinner - // on one side, and that is a data gap for that product to close. + // Load Testing now ships an entity description for every entity, so its sense of + // `project` carries a `means` too — the disambiguation reads on both sides. const lt = clarify.options.find((o: any) => o.product === "loadtesting"); expect(lt.summary).toMatch(/Load and performance testing/); expect(lt.shared_terms[0].entity).toBe("project"); - expect(lt.shared_terms[0].means).toBeUndefined(); + expect(lt.shared_terms[0].means).toMatch(/groups related load tests/); }); it("folds plurals on both sides, or the gate misses the case it was built for", async () => { diff --git a/tests/tools/capabilityRegistryRouting.test.ts b/tests/tools/capabilityRegistryRouting.test.ts index b4dc9026..5597401b 100644 --- a/tests/tools/capabilityRegistryRouting.test.ts +++ b/tests/tools/capabilityRegistryRouting.test.ts @@ -53,25 +53,26 @@ describe("routing between products", () => { // project. The only other way to find out is describeEntity, once per entity — for // tm that is 19 calls at ~1.4KB each, paid exactly when the agent is least oriented. const vocab = vocabularyOf(BOTH); - const described = vocab[tm.name].filter((e) => e.description); - expect(described.length).toBe(vocab[tm.name].length); + // Both products now author an entity description for every entity — the field + // listProducts routes on. (loadtesting previously shipped 9 entities with 0 + // descriptions; that gap is closed.) + for (const product of [tm, lt]) { + const entries = vocab[product.name]; + const described = entries.filter((e) => e.description); + expect(described.length, product.name).toBe(entries.length); - for (const entry of described) { - // One line, capped by the build. Long enough to define, short enough that every - // entity of every product can travel on one listProducts call. - expect(entry.description!.length, entry.entity).toBeLessThanOrEqual(140); - // A definition, not a restatement of the name: `tag: "tag"` would pass a presence - // check and teach nothing. - expect( - entry.description!.toLowerCase().replace(/[^a-z]/g, ""), - entry.entity, - ).not.toBe(entry.entity.replace(/[^a-z]/g, "")); + for (const entry of described) { + // One line, capped by the build. Long enough to define, short enough that every + // entity of every product can travel on one listProducts call. + expect(entry.description!.length, entry.entity).toBeLessThanOrEqual(140); + // A definition, not a restatement of the name: `tag: "tag"` would pass a presence + // check and teach nothing. + expect( + entry.description!.toLowerCase().replace(/[^a-z]/g, ""), + entry.entity, + ).not.toBe(entry.entity.replace(/[^a-z]/g, "")); + } } - - // Absent, not empty, where a product has not authored them. loadtesting ships 9 - // entities and 0 descriptions today; that has to read as "unwritten" rather than as - // a build that produced nothing. - expect(vocab[lt.name].every((e) => e.description === undefined)).toBe(true); }); it("answers within one product, so size cannot decide the answer", () => { From b652f20edacad4adf75e5dfb1fcd01675c9eaf83 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Mon, 28 Sep 2026 13:15:39 +0530 Subject: [PATCH 2/7] fix(loadtesting): guide the agent to present run duration as minutes + seconds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The report returns durationSec in whole seconds. Add guidance on getLoadTestRunReport (and a durationSec schema description) telling the agent to render it to the user as minutes + seconds (e.g. 330 -> '5m 30s'), keeping the raw seconds only for calculations. Presentation-only — no BE/unit change. --- capability/loadtesting.capability-index.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index 818e8bcc..cba36acd 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -819,7 +819,8 @@ "metrics accepts dotted names or @ aliases; see getLoadTestMetricsManifest for what applies to this test type.", "Answer a scoped question with the narrowest slice instead of the whole report: one metric → metrics=; error breakdown → errorCategory= (or slaOnly=true); a time window → sinceIso/untilIso; slowest transaction → detail=per-txn with groupBy=transaction and topN.", "One call with the right params returns everything for that question — do not re-fetch the same run with the same params. Fetch the full report (view=full / detail=full) only when the user explicitly asks for the raw or complete report.", - "A plain 'summarize this run' / 'give me a summary' request is answered by getLoadTestInsightSummary, NOT this endpoint. Only call this when the user asks for specific KPIs, SLA verdicts, web vitals, transactions, or the raw/complete report." + "A plain 'summarize this run' / 'give me a summary' request is answered by getLoadTestInsightSummary, NOT this endpoint. Only call this when the user asks for specific KPIs, SLA verdicts, web vitals, transactions, or the raw/complete report.", + "durationSec is the run's wall-clock length in WHOLE SECONDS. When presenting it to the user, render it as minutes and seconds (e.g. 330 -> '5m 30s', 90 -> '1m 30s', 45 -> '45s'), not as a raw seconds count; keep the raw seconds only for calculations." ], "returns": [ "runId", @@ -1991,7 +1992,8 @@ "type": "string" }, "durationSec": { - "type": "number" + "type": "number", + "description": "Run wall-clock duration in whole seconds. Present to the user as minutes + seconds (e.g. 5m 30s)." }, "kpis": { "type": "object" From 79e1d7e0c9a5943b313a534af31974e755a55435 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Mon, 28 Sep 2026 14:58:51 +0530 Subject: [PATCH 3/7] fix(loadtesting): complete compareLoadTestRuns contract (same-test intent + missing returns) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit compareLoadTestRuns is the regression check for runs of the SAME test, but the contract said only 'compare two completed runs' and declared just kpiDeltas/transactionDeltas — while the endpoint also returns baseline, candidate (each with testId), slaVerdictChanges and a warnings[] array. Most importantly the BE's 'runs are of different tests — comparison may be misleading' signal lives in warnings, so the agent never saw it and would present a cross- test delta as valid. - intent: state it compares runs of the SAME test; cross-test is allowed but flagged - returns + CompareLoadTestRunsData schema: add baseline, candidate, slaVerdictChanges, warnings (baseline/candidate carry runId/testId/startedAt) - guidance: prefer same-test runs; always surface warnings before reporting --- capability/loadtesting.capability-index.json | 53 ++++++++++++++++++-- 1 file changed, 50 insertions(+), 3 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index cba36acd..ef75064a 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -1098,15 +1098,21 @@ "description": "Minimum percent change to include." } ], - "intent": "Compare two completed runs and surface the deltas — use this for 'did this run regress vs the baseline?'", + "intent": "Compare two completed runs of the SAME test and surface the deltas — the regression check ('did this run regress vs the baseline?'). Comparing runs of different tests is allowed but flagged in warnings as potentially misleading.", "guidance": [ "baselineRunId and candidateRunId are run UUIDs and must differ; both runs must be terminal.", "regressedOnly + pctChangeMin filter to material regressions; dimensions accepts kpi, transaction, sla.", - "Use this to compare two runs — it returns per-KPI and per-transaction deltas directly. Do not fetch both run reports and diff them yourself." + "Use this to compare two runs — it returns per-KPI and per-transaction deltas directly. Do not fetch both run reports and diff them yourself.", + "Prefer two runs of the SAME test (pick both from listLoadTestRuns for one testId). baseline.testId and candidate.testId tell you whether they match.", + "Always check warnings before reporting the result: it flags different-test comparisons ('runs are of different tests — comparison may be misleading'), differing SLA definitions, and non-terminal runs. Surface any warning to the user — a cross-test delta can be meaningless." ], "returns": [ + "baseline", + "candidate", "kpiDeltas", - "transactionDeltas" + "transactionDeltas", + "slaVerdictChanges", + "warnings" ], "responses": { "200": { @@ -2068,11 +2074,52 @@ "CompareLoadTestRunsData": { "type": "object", "properties": { + "baseline": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "format": "uuid" + }, + "testId": { + "type": "integer" + }, + "startedAt": { + "type": "string", + "format": "date-time" + } + } + }, + "candidate": { + "type": "object", + "properties": { + "runId": { + "type": "string", + "format": "uuid" + }, + "testId": { + "type": "integer" + }, + "startedAt": { + "type": "string", + "format": "date-time" + } + } + }, "kpiDeltas": { "description": "Per-KPI baseline→candidate deltas." }, "transactionDeltas": { "type": "array" + }, + "slaVerdictChanges": { + "type": "array" + }, + "warnings": { + "type": "array", + "items": { + "type": "string" + } } } }, From 063fa164384a277fbf8740b85a8c58636a640c75 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Mon, 28 Sep 2026 16:10:36 +0530 Subject: [PATCH 4/7] fix(loadtesting): compareLoadTestRuns is same-test only, matching the BE The BE compare now mirrors the Compare Runs page: runs of different tests are rejected (DIFFERENT_TESTS), both runs must be finished and share test types, and deltas come from the page's own data sources. - intent: two executions of one test; not for different tests; use getLoadTestHistoricalTrends for more than two runs - guidance: pick both runs from listLoadTestRuns (status=terminal), older = baseline, "latest" = latest vs previous finished run; signed pctChange; how the transaction metric is chosen; SLA-changed warning - drop the "check baseline.testId == candidate.testId" advice (testId is a config version; runs across an edit legitimately differ) - groupBy values (transaction/url/label/threadGroup/scenario); corrected metrics / regressedOnly / pctChangeMin descriptions; 404 response - typed kpiDeltas / transactionDeltas / slaVerdictChanges rows - listProjectLoadTestRuns no longer suggests cross-test comparison --- capability/loadtesting.capability-index.json | 142 +++++++++++++++++-- 1 file changed, 128 insertions(+), 14 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index ef75064a..4c94d86d 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -470,7 +470,7 @@ "description": "Filter to runs carrying this tag (a run's own tags, or the tags it inherits from its test when it has none of its own)." } ], - "intent": "List the execution history across every load test in a project — newest first, paginated. Use for 'what ran last week in this project?' or to find runs across tests to compare.", + "intent": "List the execution history across every load test in a project — newest first, paginated. Use for 'what ran last week in this project?'.", "guidance": [ "Returns run metadata only, not metrics — use getLoadTestRunReport for a run's KPIs.", "Each run carries a tags array — its own run-level tags if set (via updateLoadTest with runId), otherwise the tags inherited from its test. Filter with the tag query param.", @@ -1075,12 +1075,19 @@ { "name": "metrics", "type": "string", - "description": "Comma-separated dotted metric names or @ aliases (e.g. @vitals, @all). Resolve valid names via getLoadTestMetricsManifest." + "description": "Comma-separated dotted metric names or @ aliases (e.g. latency.p95, @vitals). Filters kpiDeltas; the first entry with a per-transaction value also picks the transaction metric. Resolve valid names via getLoadTestMetricsManifest." }, { "name": "groupBy", "type": "string", - "description": "Rollup dimension for transaction deltas." + "description": "Axis for transaction deltas (default transaction = request name).", + "values": [ + "transaction", + "url", + "label", + "threadGroup", + "scenario" + ] }, { "name": "topN", @@ -1090,21 +1097,24 @@ { "name": "regressedOnly", "type": "boolean", - "description": "Return only regressed rows." + "description": "Return only regressed transaction rows." }, { "name": "pctChangeMin", "type": "number", - "description": "Minimum percent change to include." + "description": "Minimum absolute percent change for a KPI or transaction row to be included." } ], - "intent": "Compare two completed runs of the SAME test and surface the deltas — the regression check ('did this run regress vs the baseline?'). Comparing runs of different tests is allowed but flagged in warnings as potentially misleading.", + "intent": "Compare two executions (runs) of the SAME load test and surface the deltas — the regression check ('did the latest run regress vs the previous one?'). Not for comparing different tests: runs of different tests are rejected (DIFFERENT_TESTS). For a trend across more than two runs use getLoadTestHistoricalTrends.", "guidance": [ - "baselineRunId and candidateRunId are run UUIDs and must differ; both runs must be terminal.", - "regressedOnly + pctChangeMin filter to material regressions; dimensions accepts kpi, transaction, sla.", - "Use this to compare two runs — it returns per-KPI and per-transaction deltas directly. Do not fetch both run reports and diff them yourself.", - "Prefer two runs of the SAME test (pick both from listLoadTestRuns for one testId). baseline.testId and candidate.testId tell you whether they match.", - "Always check warnings before reporting the result: it flags different-test comparisons ('runs are of different tests — comparison may be misleading'), differing SLA definitions, and non-terminal runs. Surface any warning to the user — a cross-test delta can be meaningless." + "Pick both runs from listLoadTestRuns for ONE testId, filtered to finished runs (status=terminal). The older run is the baseline, the newer the candidate.", + "'Compare the latest run' means the latest finished run (candidate) vs the finished run before it (baseline).", + "Both runs must be finished and must share the same test types — otherwise the call fails with VALIDATION_ERROR. Runs of different tests fail with DIFFERENT_TESTS; do not retry with other tests' runs.", + "Use this to compare two runs — it returns per-KPI, per-transaction and SLA-verdict deltas from the same data as the Compare Runs page. Do not fetch both run reports and diff them yourself.", + "pctChange is signed (candidate vs baseline); direction/regressed already account for whether higher or lower is better — report those, do not re-derive them.", + "Transaction deltas are computed on one metric: the first metrics entry that has a per-transaction value (latency.*, errors.rate, throughput.*), else latency.p95. groupBy picks the transaction axis.", + "regressedOnly + pctChangeMin filter to material regressions; regressedOnly applies to transactionDeltas, pctChangeMin to kpiDeltas and transactionDeltas.", + "Check warnings before reporting: it flags SLA definitions that changed between the runs, which can flip verdicts without a performance change. Surface any warning to the user." ], "returns": [ "baseline", @@ -1137,6 +1147,9 @@ "401": { "$response": "Unauthorized" }, + "404": { + "$response": "NotFound" + }, "500": { "$response": "InternalServerError" } @@ -2107,13 +2120,114 @@ } }, "kpiDeltas": { - "description": "Per-KPI baseline→candidate deltas." + "type": "array", + "description": "Per-KPI baseline→candidate deltas.", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string" + }, + "baseline": { + "type": "number", + "description": "Null when either run lacks the value." + }, + "candidate": { + "type": "number", + "description": "Null when either run lacks the value." + }, + "absChange": { + "type": "number", + "description": "Null when either run lacks the value." + }, + "pctChange": { + "type": "number", + "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value." + }, + "direction": { + "type": "string", + "enum": [ + "improved", + "regressed", + "unchanged" + ] + } + } + } }, "transactionDeltas": { - "type": "array" + "type": "array", + "description": "Per-transaction deltas on one metric, sorted by |pctChange| desc.", + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "metric": { + "type": "string" + }, + "baseline": { + "type": "number", + "description": "Null when either run lacks the value." + }, + "candidate": { + "type": "number", + "description": "Null when either run lacks the value." + }, + "absChange": { + "type": "number", + "description": "Null when either run lacks the value." + }, + "pctChange": { + "type": "number", + "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value." + }, + "regressed": { + "type": "boolean" + } + } + } }, "slaVerdictChanges": { - "type": "array" + "type": "array", + "description": "SLA thresholds whose verdict changed between the runs.", + "items": { + "type": "object", + "properties": { + "metric": { + "type": "string" + }, + "type": { + "type": "string" + }, + "condition": { + "type": "string" + }, + "threshold": {}, + "unit": { + "type": "string" + }, + "request": {}, + "from": { + "type": "string", + "description": "Null when the threshold is absent on that run." + }, + "to": { + "type": "string", + "description": "Null when the threshold is absent on that run." + }, + "diffStatus": { + "type": "string", + "enum": [ + "now_failing", + "now_passing", + "added", + "dropped" + ] + } + } + } }, "warnings": { "type": "array", From 3d1833a569e8c74bd0686aaf56ac120bd0f3a6c4 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Tue, 29 Sep 2026 12:37:47 +0530 Subject: [PATCH 5/7] fix(loadtesting): drop url from compareLoadTestRuns groupBy; document the unservable-metrics warning Follows load-testing-backend#3135 review: the BE no longer offers a raw url axis for transaction deltas (url rows are keyed by request name, which merged distinct URLs), matching the Compare Runs page allowlist. Warnings now also list requested metrics compare cannot serve. --- capability/loadtesting.capability-index.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index 4c94d86d..83011571 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -1083,7 +1083,6 @@ "description": "Axis for transaction deltas (default transaction = request name).", "values": [ "transaction", - "url", "label", "threadGroup", "scenario" @@ -1114,7 +1113,7 @@ "pctChange is signed (candidate vs baseline); direction/regressed already account for whether higher or lower is better — report those, do not re-derive them.", "Transaction deltas are computed on one metric: the first metrics entry that has a per-transaction value (latency.*, errors.rate, throughput.*), else latency.p95. groupBy picks the transaction axis.", "regressedOnly + pctChangeMin filter to material regressions; regressedOnly applies to transactionDeltas, pctChangeMin to kpiDeltas and transactionDeltas.", - "Check warnings before reporting: it flags SLA definitions that changed between the runs, which can flip verdicts without a performance change. Surface any warning to the user." + "Check warnings before reporting: it flags SLA definitions that changed between the runs (verdicts can flip without a performance change) and lists requested metrics compare cannot serve (e.g. engine.*). Surface any warning to the user." ], "returns": [ "baseline", From acd24761dabc1f21ee188c822c7945aedcb8e437 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Tue, 29 Sep 2026 16:41:15 +0530 Subject: [PATCH 6/7] fix(loadtesting): compareLoadTestRuns direction can be unknown; document the new warnings Follows load-testing-backend#3135 review round 2: a metric missing on either run now reports direction "unknown" (regressed null) instead of "unchanged"; warnings also name no-data metrics and runs whose SLAs were not evaluated. --- capability/loadtesting.capability-index.json | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index 83011571..939348be 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -1110,10 +1110,10 @@ "'Compare the latest run' means the latest finished run (candidate) vs the finished run before it (baseline).", "Both runs must be finished and must share the same test types — otherwise the call fails with VALIDATION_ERROR. Runs of different tests fail with DIFFERENT_TESTS; do not retry with other tests' runs.", "Use this to compare two runs — it returns per-KPI, per-transaction and SLA-verdict deltas from the same data as the Compare Runs page. Do not fetch both run reports and diff them yourself.", - "pctChange is signed (candidate vs baseline); direction/regressed already account for whether higher or lower is better — report those, do not re-derive them.", + "pctChange is signed (candidate vs baseline); direction/regressed already account for whether higher or lower is better — report those, do not re-derive them. direction \"unknown\" (regressed null) means one or both runs have no value for that metric: report it as missing data, never as no change.", "Transaction deltas are computed on one metric: the first metrics entry that has a per-transaction value (latency.*, errors.rate, throughput.*), else latency.p95. groupBy picks the transaction axis.", "regressedOnly + pctChangeMin filter to material regressions; regressedOnly applies to transactionDeltas, pctChangeMin to kpiDeltas and transactionDeltas.", - "Check warnings before reporting: it flags SLA definitions that changed between the runs (verdicts can flip without a performance change) and lists requested metrics compare cannot serve (e.g. engine.*). Surface any warning to the user." + "Check warnings before reporting: they name metrics with no data on a run, runs whose SLA thresholds were not evaluated, SLA definitions that changed between the runs, and requested metrics compare cannot serve (e.g. engine.*). Surface any warning to the user." ], "returns": [ "baseline", @@ -2148,8 +2148,10 @@ "enum": [ "improved", "regressed", - "unchanged" - ] + "unchanged", + "unknown" + ], + "description": "\"unknown\" when either run has no value for the metric." } } } @@ -2183,7 +2185,8 @@ "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value." }, "regressed": { - "type": "boolean" + "type": "boolean", + "description": "Null when either run lacks the transaction (unknown, not unregressed)." } } } From 3ccd5a630af63127ae6e73854619a08dae8da1f0 Mon Sep 17 00:00:00 2001 From: sourabhd-cbu Date: Tue, 29 Sep 2026 20:44:19 +0530 Subject: [PATCH 7/7] =?UTF-8?q?fix(loadtesting):=20address=20#450=20review?= =?UTF-8?q?=20=E2=80=94=20date=20params,=20nullability,=20row=20schemas,?= =?UTF-8?q?=20200=20prose?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - sinceIso / untilIso / updatedSince (11 params) drop the enforced date-time format: bind rejected date-only values ("2026-09-12") and timestamps without seconds before the request left, while the BE parses any new Date()-valid ISO-8601 value. Each now documents both shapes and carries an example. - version (create/update/get) declares format date-time — the BE emits updated_at via toISOString() — so the ifVersion format it round-trips into is asserted on the producer side too. - CompareLoadTestRunsData delta fields that can be null (baseline, candidate, absChange, pctChange, regressed, from, to) are marked nullable: true, the convention tm already uses. - Row arrays get item schemas from the BE shapers: tests (list + search), projects, runs (per-test, per-project, active) — backing the fields the guidance promises (testId/projectId/name/testType/framework, tags, …). - Inline 200s keep the success-envelope description; the now-unreferenced named Success response is removed. - types.ts: EntityDoc comment no longer claims loadtesting authors 0 of 9. --- capability/loadtesting.capability-index.json | 388 ++++++++++++++++--- src/tools/capability-registry/types.ts | 4 +- 2 files changed, 339 insertions(+), 53 deletions(-) diff --git a/capability/loadtesting.capability-index.json b/capability/loadtesting.capability-index.json index 939348be..7eed1e56 100644 --- a/capability/loadtesting.capability-index.json +++ b/capability/loadtesting.capability-index.json @@ -28,6 +28,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -100,6 +101,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -188,9 +190,8 @@ { "name": "updatedSince", "type": "string", - "description": "ISO-8601 timestamp; only tests updated after it.", - "example": "2026-08-01T00:00:00Z", - "format": "date-time" + "description": "ISO-8601 timestamp; only tests updated after it. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-08-01T00:00:00Z" } ], "intent": "List the load tests in a project — use this to find a testId or browse tests, optionally filtered by type, framework, tag or recent activity.", @@ -208,6 +209,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -273,6 +275,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -344,15 +347,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound.", - "example": "2026-08-01T00:00:00Z", - "format": "date-time" + "description": "ISO-8601 lower bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-08-01T00:00:00Z" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound.", - "format": "date-time" + "description": "ISO-8601 upper bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" } ], "intent": "Show how a project's load-test metrics trend across recent runs — use this for 'is performance getting better or worse across this project?'", @@ -366,6 +368,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -449,15 +452,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound.", - "example": "2026-08-01T00:00:00Z", - "format": "date-time" + "description": "ISO-8601 lower bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-08-01T00:00:00Z" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound.", - "format": "date-time" + "description": "ISO-8601 upper bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" }, { "name": "hadSlaBreach", @@ -485,6 +487,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -563,6 +566,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -627,6 +631,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -696,6 +701,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -801,14 +807,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound; slices the report to data in this time window.", - "format": "date-time" + "description": "ISO-8601 lower bound; slices the report to data in this time window. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound for the time-window slice.", - "format": "date-time" + "description": "ISO-8601 upper bound for the time-window slice. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" } ], "intent": "Get the full metrics report for a completed run — KPIs, SLA verdicts, per-transaction and error breakdowns. Use this to answer 'how did this run perform?'", @@ -836,6 +842,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -891,6 +898,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -938,6 +946,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1013,6 +1022,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1125,6 +1135,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1269,6 +1280,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1364,6 +1376,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1432,14 +1445,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound.", - "format": "date-time" + "description": "ISO-8601 lower bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound.", - "format": "date-time" + "description": "ISO-8601 upper bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" }, { "name": "verdict", @@ -1478,6 +1491,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1554,14 +1568,14 @@ { "name": "sinceIso", "type": "string", - "description": "ISO-8601 lower bound.", - "format": "date-time" + "description": "ISO-8601 lower bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" }, { "name": "untilIso", "type": "string", - "description": "ISO-8601 upper bound.", - "format": "date-time" + "description": "ISO-8601 upper bound. ISO-8601 — a date (2026-09-12) or a date-time (2026-09-12T10:30:00Z).", + "example": "2026-09-12" } ], "intent": "Show how one test's metrics trend across its recent runs — use this for 'is this test getting slower over time?'", @@ -1575,6 +1589,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1629,6 +1644,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1731,6 +1747,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1806,6 +1823,7 @@ ], "responses": { "200": { + "description": "Standard success envelope; operation data under `data`.", "schema": { "allOf": [ { @@ -1894,7 +1912,39 @@ "type": "object", "properties": { "projects": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "projectId": { + "type": "integer" + }, + "name": { + "type": "string" + }, + "testCount": { + "type": "integer", + "nullable": true + }, + "lastActivityAt": { + "type": "string", + "nullable": true + }, + "owner": { + "type": "object", + "properties": { + "userId": { + "type": "integer", + "nullable": true + }, + "email": { + "type": "string", + "nullable": true + } + } + } + } + } }, "hasMore": { "type": "boolean" @@ -1908,7 +1958,68 @@ "type": "object", "properties": { "tests": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "testId": { + "type": "integer" + }, + "name": { + "type": "string", + "nullable": true + }, + "testType": { + "type": "string", + "description": "plu (API), blu (Browser) or hybrid." + }, + "framework": { + "type": "string", + "nullable": true + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "owner": { + "type": "object", + "properties": { + "userId": { + "type": "integer", + "nullable": true + }, + "email": { + "type": "string", + "nullable": true + } + } + }, + "lastRun": { + "type": "object", + "description": "Most recent execution, when present.", + "properties": { + "runId": { + "type": "string", + "nullable": true + }, + "status": { + "type": "string", + "nullable": true + }, + "startedAt": { + "type": "string", + "nullable": true + }, + "verdict": { + "type": "string", + "nullable": true + } + } + } + } + } }, "hasMore": { "type": "boolean" @@ -1922,7 +2033,30 @@ "type": "object", "properties": { "tests": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "testId": { + "type": "integer" + }, + "name": { + "type": "string", + "nullable": true + }, + "projectId": { + "type": "integer", + "nullable": true + }, + "testType": { + "type": "string" + }, + "framework": { + "type": "string", + "nullable": true + } + } + } }, "hasMore": { "type": "boolean" @@ -1941,7 +2075,48 @@ "type": "object", "properties": { "runs": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "runId": { + "type": "string" + }, + "status": { + "type": "string", + "nullable": true + }, + "startedAt": { + "type": "string", + "nullable": true + }, + "durationSec": { + "type": "integer", + "nullable": true, + "description": "Wall-clock seconds; null while running or when timestamps are missing." + }, + "vus": { + "type": "integer", + "nullable": true + }, + "verdict": { + "type": "string", + "nullable": true + }, + "vuHours": { + "type": "number", + "nullable": true, + "description": "Billed VU-hours; null until billed." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Run-level tags, or the tags inherited from the test when the run has none." + } + } + } }, "hasMore": { "type": "boolean" @@ -1955,7 +2130,66 @@ "type": "object", "properties": { "runs": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "runId": { + "type": "string" + }, + "testId": { + "type": "integer", + "nullable": true + }, + "testName": { + "type": "string", + "nullable": true + }, + "projectId": { + "type": "integer", + "nullable": true + }, + "startedBy": { + "type": "object", + "properties": { + "userId": { + "type": "integer", + "nullable": true + }, + "email": { + "type": "string", + "nullable": true + } + } + }, + "startedAt": { + "type": "string", + "nullable": true + }, + "elapsedSec": { + "type": "integer", + "nullable": true + }, + "vus": { + "type": "integer", + "nullable": true + }, + "vuHoursBurnedSoFar": { + "type": "number" + }, + "status": { + "type": "string", + "nullable": true + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Run-level tags, or the tags inherited from the test when the run has none." + } + } + } } } }, @@ -2129,19 +2363,23 @@ }, "baseline": { "type": "number", - "description": "Null when either run lacks the value." + "description": "Null when either run lacks the value.", + "nullable": true }, "candidate": { "type": "number", - "description": "Null when either run lacks the value." + "description": "Null when either run lacks the value.", + "nullable": true }, "absChange": { "type": "number", - "description": "Null when either run lacks the value." + "description": "Null when either run lacks the value.", + "nullable": true }, "pctChange": { "type": "number", - "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value." + "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value.", + "nullable": true }, "direction": { "type": "string", @@ -2170,23 +2408,28 @@ }, "baseline": { "type": "number", - "description": "Null when either run lacks the value." + "description": "Null when either run lacks the value.", + "nullable": true }, "candidate": { "type": "number", - "description": "Null when either run lacks the value." + "description": "Null when either run lacks the value.", + "nullable": true }, "absChange": { "type": "number", - "description": "Null when either run lacks the value." + "description": "Null when either run lacks the value.", + "nullable": true }, "pctChange": { "type": "number", - "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value." + "description": "Signed percent change, candidate vs baseline. Null when either run lacks the value.", + "nullable": true }, "regressed": { "type": "boolean", - "description": "Null when either run lacks the transaction (unknown, not unregressed)." + "description": "Null when either run lacks the transaction (unknown, not unregressed).", + "nullable": true } } } @@ -2213,11 +2456,13 @@ "request": {}, "from": { "type": "string", - "description": "Null when the threshold is absent on that run." + "description": "Null when the threshold is absent on that run.", + "nullable": true }, "to": { "type": "string", - "description": "Null when the threshold is absent on that run." + "description": "Null when the threshold is absent on that run.", + "nullable": true }, "diffStatus": { "type": "string", @@ -2252,7 +2497,9 @@ "type": "integer" }, "version": { - "type": "string" + "type": "string", + "format": "date-time", + "description": "Optimistic-concurrency token: the test's updated_at as an ISO-8601 date-time (seconds precision). Pass it back unchanged as updateLoadTest's ifVersion." }, "dashboardLink": { "type": "string" @@ -2285,7 +2532,48 @@ "type": "object", "properties": { "runs": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "runId": { + "type": "string" + }, + "status": { + "type": "string", + "nullable": true + }, + "startedAt": { + "type": "string", + "nullable": true + }, + "durationSec": { + "type": "integer", + "nullable": true, + "description": "Wall-clock seconds; null while running or when timestamps are missing." + }, + "vus": { + "type": "integer", + "nullable": true + }, + "verdict": { + "type": "string", + "nullable": true + }, + "vuHours": { + "type": "number", + "nullable": true, + "description": "Billed VU-hours; null until billed." + }, + "tags": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Run-level tags, or the tags inherited from the test when the run has none." + } + } + } }, "hasMore": { "type": "boolean" @@ -2324,7 +2612,9 @@ "type": "integer" }, "version": { - "type": "string" + "type": "string", + "format": "date-time", + "description": "Optimistic-concurrency token: the test's updated_at as an ISO-8601 date-time (seconds precision). Pass it back unchanged as updateLoadTest's ifVersion." } } }, @@ -2344,7 +2634,9 @@ "type": "string" }, "version": { - "type": "string" + "type": "string", + "format": "date-time", + "description": "Optimistic-concurrency token: the test's updated_at as an ISO-8601 date-time (seconds precision). Pass it back unchanged as updateLoadTest's ifVersion." }, "projectId": { "type": "integer" @@ -2359,12 +2651,6 @@ } }, "responses": { - "Success": { - "description": "Standard success envelope; operation data under `data`.", - "schema": { - "$schema": "SuccessEnvelope" - } - }, "BadRequest": { "description": "Invalid parameters or body.", "schema": { diff --git a/src/tools/capability-registry/types.ts b/src/tools/capability-registry/types.ts index 524138ed..33fd2131 100644 --- a/src/tools/capability-registry/types.ts +++ b/src/tools/capability-registry/types.ts @@ -175,8 +175,8 @@ export interface EntityDoc { * answer it for all of them at once. * * Capped at 140 characters by the build, and dropped rather than truncated when longer. - * Often absent: a product that has not authored these emits none (loadtesting has 0 of - * 9 today), and absence means "unwritten", never an error. + * Often absent: a product that has not authored these emits none (loadtesting authors + * all 9 of its entities), and absence means "unwritten", never an error. */ description?: string; aliases?: string[];