From 22f39584e082115845a537cc6c0f5999ac852147 Mon Sep 17 00:00:00 2001 From: Simon Siju Date: Mon, 8 Jun 2026 15:19:04 -0400 Subject: [PATCH 1/2] Add is_first_party label to CLI command metrics Adds an is_first_party label to cli_commands_total (and the duration / wall-clock-elapsed metrics) so first-party (1P) Shopify developer usage can be distinguished from third-party usage in observability dashboards. The label is keyed off the 1P dev path (SHOPIFY_CLI_1P_DEV) via firstPartyDev(), the same signal that sets the X-Shopify-Cli-Employee header. This unblocks the 1P CLI guardrail panels on the [Dev Platform] 1P Orgs Migration dashboard, which currently blend 1P + 3P traffic. Closes: https://github.com/shop/issues-develop/issues/22933 --- .changeset/first-party-cli-metrics-label.md | 5 +++ .../__snapshots__/otel-metrics.test.ts.snap | 11 ++++- .../src/private/node/otel-metrics.test.ts | 40 ++++++++++++++++++- .../cli-kit/src/private/node/otel-metrics.ts | 7 +++- 4 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 .changeset/first-party-cli-metrics-label.md diff --git a/.changeset/first-party-cli-metrics-label.md b/.changeset/first-party-cli-metrics-label.md new file mode 100644 index 00000000000..596bd05aa1c --- /dev/null +++ b/.changeset/first-party-cli-metrics-label.md @@ -0,0 +1,5 @@ +--- +'@shopify/cli-kit': patch +--- + +Add an `is_first_party` label to the CLI command metrics (`cli_commands_total`, `cli_commands_duration_ms`, `cli_commands_wall_clock_elapsed_ms`) so first-party (1P) Shopify developer usage can be distinguished from third-party usage in observability dashboards. The label is keyed off the 1P dev path (`SHOPIFY_CLI_1P_DEV`), the same signal that sets the `X-Shopify-Cli-Employee` header. diff --git a/packages/cli-kit/src/private/node/__snapshots__/otel-metrics.test.ts.snap b/packages/cli-kit/src/private/node/__snapshots__/otel-metrics.test.ts.snap index 3a1d3fdc918..45151db9f5b 100644 --- a/packages/cli-kit/src/private/node/__snapshots__/otel-metrics.test.ts.snap +++ b/packages/cli-kit/src/private/node/__snapshots__/otel-metrics.test.ts.snap @@ -8,6 +8,7 @@ exports[`otel-metrics > logs metrics when activated 1`] = ` { "cli_version": "pre", "exit": "ok", + "is_first_party": "false", "job": "@shopify/app::app dev", }, ], @@ -17,6 +18,7 @@ exports[`otel-metrics > logs metrics when activated 1`] = ` { "cli_version": "pre", "exit": "ok", + "is_first_party": "false", "job": "@shopify/app::app dev", }, ], @@ -26,6 +28,7 @@ exports[`otel-metrics > logs metrics when activated 1`] = ` { "cli_version": "pre", "exit": "ok", + "is_first_party": "false", "job": "@shopify/app::app dev", "stage": "active", }, @@ -36,6 +39,7 @@ exports[`otel-metrics > logs metrics when activated 1`] = ` { "cli_version": "pre", "exit": "ok", + "is_first_party": "false", "job": "@shopify/app::app dev", "stage": "network", }, @@ -46,6 +50,7 @@ exports[`otel-metrics > logs metrics when activated 1`] = ` { "cli_version": "pre", "exit": "ok", + "is_first_party": "false", "job": "@shopify/app::app dev", "stage": "prompt", }, @@ -58,14 +63,16 @@ exports[`otel-metrics > outputs debug information when deactivated 1`] = ` "labels": { "exit": "ok", "job": "@shopify/app::app dev", - "cli_version": "nightly" + "cli_version": "nightly", + "is_first_party": "false" } } [OTEL] record cli_commands_duration_ms histogram 10ms { "labels": { "exit": "ok", "job": "@shopify/app::app dev", - "cli_version": "nightly" + "cli_version": "nightly", + "is_first_party": "false" } } [OTEL] record cli_commands_wall_clock_elapsed_ms histogram stage="active" 10ms diff --git a/packages/cli-kit/src/private/node/otel-metrics.test.ts b/packages/cli-kit/src/private/node/otel-metrics.test.ts index 0209846d62a..a8855ae5c79 100644 --- a/packages/cli-kit/src/private/node/otel-metrics.test.ts +++ b/packages/cli-kit/src/private/node/otel-metrics.test.ts @@ -1,8 +1,12 @@ import {recordMetrics} from './otel-metrics.js' import {mockAndCaptureOutput} from '../../public/node/testing/output.js' -import {describe, expect, test, vi} from 'vitest' +import {afterEach, describe, expect, test, vi} from 'vitest' describe('otel-metrics', () => { + afterEach(() => { + vi.unstubAllEnvs() + }) + test('outputs debug information when deactivated', async () => { const outputMock = mockAndCaptureOutput() @@ -53,4 +57,38 @@ describe('otel-metrics', () => { expect(mockOtelCreator).toHaveBeenCalledOnce() expect(mockOtelRecorder.mock.calls).toMatchSnapshot() }) + + test('labels metrics as first-party when the 1P dev path is enabled', async () => { + vi.stubEnv('SHOPIFY_CLI_1P_DEV', '1') + const mockOtelRecorder = vi.fn() + const mockOtelCreator = vi.fn() + mockOtelCreator.mockReturnValue({ + type: 'otel', + otel: { + record: mockOtelRecorder, + }, + }) + + await recordMetrics( + { + skipMetricAnalytics: false, + cliVersion: '3.49.1-pre.0', + owningPlugin: '@shopify/app', + command: 'app dev', + exitMode: 'ok', + }, + { + active: 10, + network: 20, + prompt: 30, + }, + mockOtelCreator, + ) + + const recordedLabels = mockOtelRecorder.mock.calls.map((call) => call[2]) + expect(recordedLabels.length).toBeGreaterThan(0) + recordedLabels.forEach((labels) => { + expect(labels).toMatchObject({is_first_party: 'true'}) + }) + }) }) diff --git a/packages/cli-kit/src/private/node/otel-metrics.ts b/packages/cli-kit/src/private/node/otel-metrics.ts index 5e0545503a9..dd1d1a1dba6 100644 --- a/packages/cli-kit/src/private/node/otel-metrics.ts +++ b/packages/cli-kit/src/private/node/otel-metrics.ts @@ -4,7 +4,7 @@ import { DefaultOtelService, DefaultOtelServiceOptions, } from '../../public/node/vendor/otel-js/service/DefaultOtelService/DefaultOtelService.js' -import {isUnitTest, opentelemetryDomain} from '../../public/node/context/local.js' +import {firstPartyDev, isUnitTest, opentelemetryDomain} from '../../public/node/context/local.js' import {ValueType, diag} from '@opentelemetry/api' type MetricRecorder = @@ -20,6 +20,7 @@ type Labels = { exit: string job: string cli_version: string + is_first_party: string } interface Timing { @@ -76,6 +77,10 @@ export async function recordMetrics( exit: options.exitMode, job: `${options.owningPlugin}::${options.command}`, cli_version: regularisedCliVersion, + // Distinguishes Shopify first-party (1P) developers from third-party developers so the + // CLI guardrail dashboards can be scoped to 1P. Keyed off the 1P dev path + // (SHOPIFY_CLI_1P_DEV), the same signal that sets the X-Shopify-Cli-Employee header. + is_first_party: firstPartyDev() ? 'true' : 'false', } recordCommandCounter(recorder, labels) From 8bbd8c2fa590db642d0c989501de556bca3d5b45 Mon Sep 17 00:00:00 2001 From: Simon Siju Date: Mon, 8 Jun 2026 15:36:07 -0400 Subject: [PATCH 2/2] Remove changeset for CLI metrics label --- .changeset/first-party-cli-metrics-label.md | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 .changeset/first-party-cli-metrics-label.md diff --git a/.changeset/first-party-cli-metrics-label.md b/.changeset/first-party-cli-metrics-label.md deleted file mode 100644 index 596bd05aa1c..00000000000 --- a/.changeset/first-party-cli-metrics-label.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@shopify/cli-kit': patch ---- - -Add an `is_first_party` label to the CLI command metrics (`cli_commands_total`, `cli_commands_duration_ms`, `cli_commands_wall_clock_elapsed_ms`) so first-party (1P) Shopify developer usage can be distinguished from third-party usage in observability dashboards. The label is keyed off the 1P dev path (`SHOPIFY_CLI_1P_DEV`), the same signal that sets the `X-Shopify-Cli-Employee` header.