diff --git a/.changeset/bright-experiments-flags.md b/.changeset/bright-experiments-flags.md new file mode 100644 index 00000000..117be43c --- /dev/null +++ b/.changeset/bright-experiments-flags.md @@ -0,0 +1,9 @@ +--- +'flags': patch +--- + +Add the `experimental_reportOverride` adapter hook for observing values set by +the Flags SDK override cookie. + +This API is not supported for general use yet. Do not use it unless Vercel has +explicitly enabled it for you. diff --git a/.changeset/bright-experiments-report.md b/.changeset/bright-experiments-report.md new file mode 100644 index 00000000..b2405569 --- /dev/null +++ b/.changeset/bright-experiments-report.md @@ -0,0 +1,19 @@ +--- +'@vercel/flags-core': patch +--- + +Add APIs for reporting flag exposures and override values. + +- The `experimental_reportExposures` client option for supplying an exposure + reporter. +- The `experimental_reportOverride` client method for reporting values set by + the Flags SDK override cookie. +- The `experimental_exposureLogging` option on `evaluate()` and + `bulkEvaluate()` for disabling exposure reporting for an individual call. +- Experiment assignment metadata on `EvaluationResult.experiment`. +- The `experimental_EvaluationOptions`, + `experimental_ExperimentAssignment`, `experimental_Exposure`, and + `experimental_ReportExposures` types. + +These APIs are not supported for general use yet. Do not use them unless +Vercel has explicitly enabled them for you. diff --git a/.changeset/bright-experiments-vercel-adapter.md b/.changeset/bright-experiments-vercel-adapter.md new file mode 100644 index 00000000..fcbfffca --- /dev/null +++ b/.changeset/bright-experiments-vercel-adapter.md @@ -0,0 +1,9 @@ +--- +'@flags-sdk/vercel': patch +--- + +Implement the `experimental_reportOverride` adapter hook to forward override +values to the underlying Vercel Flags client. + +This API is not supported for general use yet. Do not use it unless Vercel has +explicitly enabled it for you. diff --git a/packages/adapter-vercel/src/index.test.ts b/packages/adapter-vercel/src/index.test.ts index 9920b7fc..654af1e7 100644 --- a/packages/adapter-vercel/src/index.test.ts +++ b/packages/adapter-vercel/src/index.test.ts @@ -1,4 +1,8 @@ -import { flagsClient, resetDefaultFlagsClient } from '@vercel/flags-core'; +import { + type FlagsClient, + flagsClient, + resetDefaultFlagsClient, +} from '@vercel/flags-core'; import type { Adapter, Origin, ProviderData } from 'flags'; import { flag } from 'flags/next'; import { HttpResponse, http } from 'msw'; @@ -104,6 +108,37 @@ describe('createVercelAdapter', () => { } satisfies Origin); }); + it('forwards override observations to the flags client', async () => { + const reportOverride = vi.fn(); + const fakeClient = { + origin: { provider: 'vercel', sdkKey: 'vf_x' }, + experimental_reportOverride: reportOverride, + } as unknown as typeof flagsClient; + const adapter = createVercelAdapter(fakeClient)(); + const entities = { user: { key: 'user_1' } }; + + await adapter.experimental_reportOverride?.({ + key: 'checkout', + value: 'treatment', + entities, + }); + + expect(reportOverride).toHaveBeenCalledWith({ + key: 'checkout', + value: 'treatment', + entities, + }); + }); + + it('does not expose override reporting when the flags client does not support it', () => { + const fakeClient: FlagsClient = { ...flagsClient }; + delete fakeClient.experimental_reportOverride; + + const adapter = createVercelAdapter(fakeClient)(); + + expect(adapter.experimental_reportOverride).toBeUndefined(); + }); + it('has correct types', () => { const adapter = createVercelAdapter(flagsClient); type SampleValue = boolean; diff --git a/packages/adapter-vercel/src/index.ts b/packages/adapter-vercel/src/index.ts index 6c4ef850..a88e3dce 100644 --- a/packages/adapter-vercel/src/index.ts +++ b/packages/adapter-vercel/src/index.ts @@ -40,6 +40,7 @@ export function createVercelAdapter( adapterId, origin: flagsClient.origin, config: { reportValue: false }, + experimental_reportOverride: flagsClient.experimental_reportOverride, async decide({ key, entities }) { const evaluationResult = await flagsClient.evaluate( key, diff --git a/packages/flags/src/index.test.ts b/packages/flags/src/index.test.ts index eefa3f0f..d26edc3c 100644 --- a/packages/flags/src/index.test.ts +++ b/packages/flags/src/index.test.ts @@ -27,7 +27,7 @@ describe('exports', () => { it('exports version', () => { expect(version).toBeTypeOf('string'); - expect(version).toMatch(/^\d+\.\d+\.\d+(-\w+-\d+)?$/); + expect(version).toMatch(/^\d+\.\d+\.\d+(-[\w.-]+)?$/); }); }); diff --git a/packages/flags/src/next/evaluate.ts b/packages/flags/src/next/evaluate.ts index 23ced1a4..aaa59e1d 100644 --- a/packages/flags/src/next/evaluate.ts +++ b/packages/flags/src/next/evaluate.ts @@ -40,6 +40,27 @@ const evaluationCache = new WeakMap< Map> >(); +const adapterInitializationCache = new WeakMap>(); + +async function ensureAdapterInitialized( + adapter: Pick, 'initialize'>, +): Promise { + if (!adapter.initialize) return; + + let initialization = adapterInitializationCache.get(adapter); + if (!initialization) { + initialization = adapter.initialize(); + adapterInitializationCache.set(adapter, initialization); + } + + try { + await initialization; + } catch (error) { + adapterInitializationCache.delete(adapter); + throw error; + } +} + function getCachedValuePromise( /** * supports Headers for App Router and IncomingHttpHeaders for Pages Router @@ -197,7 +218,10 @@ type FlagInfo = { key: string; defaultValue?: ValueType; config?: { reportValue?: boolean }; - adapter?: { config?: { reportValue?: boolean } }; + adapter?: Pick< + Adapter, + 'config' | 'initialize' | 'experimental_reportOverride' + >; }; function hasOverride( @@ -227,10 +251,18 @@ async function applyResult(args: { definition: FlagInfo; readonlyHeaders: ReadonlyHeaders; entitiesKey: string; + entities?: unknown; overrides: Record | null; produce: () => ValueType | PromiseLike; }): Promise { - const { definition, readonlyHeaders, entitiesKey, overrides, produce } = args; + const { + definition, + readonlyHeaders, + entitiesKey, + entities, + overrides, + produce, + } = args; const cachedValue = getCachedValuePromise( readonlyHeaders, @@ -254,6 +286,19 @@ async function applyResult(args: { internalReportValue(definition.key, decision, { reason: 'override', }); + try { + const adapter = definition.adapter; + if (adapter?.experimental_reportOverride) { + await ensureAdapterInitialized(adapter); + await adapter.experimental_reportOverride({ + key: definition.key, + value: decision, + entities, + }); + } + } catch (error) { + console.error('flags: Failed to report flag override', error); + } return decision; } @@ -401,6 +446,7 @@ export function getRun( definition, readonlyHeaders, entitiesKey, + entities, overrides, produce: () => decide({ @@ -641,6 +687,7 @@ async function evaluateImpl( definition: flagFn, readonlyHeaders, entitiesKey, + entities, overrides, produce: () => { if (bulkError) throw bulkError; diff --git a/packages/flags/src/next/index.test.ts b/packages/flags/src/next/index.test.ts index 120ac5d7..defc0ebf 100644 --- a/packages/flags/src/next/index.test.ts +++ b/packages/flags/src/next/index.test.ts @@ -191,7 +191,23 @@ describe('flag on app router', () => { it('respects overrides', async () => { const decide = vi.fn(() => false); - const f = flag({ key: 'first-flag', decide }); + const calls: string[] = []; + const initialize = vi.fn(async () => { + calls.push('initialize'); + }); + const reportOverride = vi.fn(async () => { + calls.push('reportOverride'); + }); + const entities = { user: { id: 'user_1' } }; + const f = flag({ + key: 'first-flag', + identify: () => entities, + adapter: { + decide, + initialize, + experimental_reportOverride: reportOverride, + }, + }); // first request using the flag twice const headersOfFirstRequest = new Headers(); @@ -207,6 +223,13 @@ describe('flag on app router', () => { await expect(f()).resolves.toEqual(true); expect(cookieMock).toHaveBeenCalledWith('vercel-flag-overrides'); expect(decide).not.toHaveBeenCalled(); + expect(initialize).toHaveBeenCalledOnce(); + expect(reportOverride).toHaveBeenCalledWith({ + key: 'first-flag', + value: true, + entities, + }); + expect(calls).toEqual(['initialize', 'reportOverride']); }); it('does not crash when override reporting hook is not a function', async () => { @@ -879,6 +902,10 @@ describe('evaluate', () => { bulkDecide?: Adapter['bulkDecide']; decide?: Adapter['decide']; identify?: Adapter['identify']; + experimental_reportOverride?: Adapter< + V, + any + >['experimental_reportOverride']; omitAdapterId?: boolean; omitBulkDecide?: boolean; }) { @@ -892,6 +919,7 @@ describe('evaluate', () => { throw new Error('decide should not be called in bulk path'); }), identify: opts?.identify, + experimental_reportOverride: opts?.experimental_reportOverride, ...(opts?.omitBulkDecide ? {} : { bulkDecide: opts?.bulkDecide }), }); } @@ -1084,7 +1112,13 @@ describe('evaluate', () => { it('lets overrides win over bulkDecide results', async () => { const bulkDecideMock = vi.fn().mockResolvedValue({ a: 'bulk-value' }); - const adapter = makeBulkAdapter({ bulkDecide: bulkDecideMock }); + const reportOverride = vi.fn(); + const entities = { user: { id: 'user_1' } }; + const adapter = makeBulkAdapter({ + bulkDecide: bulkDecideMock, + identify: () => entities, + experimental_reportOverride: reportOverride, + }); const a = flag({ key: 'a', adapter: adapter() }); @@ -1099,6 +1133,11 @@ describe('evaluate', () => { await expect(evaluate({ a })).resolves.toEqual({ a: true }); expect(bulkDecideMock).not.toHaveBeenCalled(); + expect(reportOverride).toHaveBeenCalledWith({ + key: 'a', + value: true, + entities, + }); }); it('omits overridden flags from bulkDecide input', async () => { diff --git a/packages/flags/src/types.ts b/packages/flags/src/types.ts index dc9563d0..f1f2cf82 100644 --- a/packages/flags/src/types.ts +++ b/packages/flags/src/types.ts @@ -165,6 +165,17 @@ export interface Adapter { * an `adapterId` are never batched. */ adapterId?: string | symbol; + /** + * Observe a value supplied by the Flags SDK override cookie. + * + * @remarks This API is not supported for general use yet. Do not use it + * unless Vercel has explicitly enabled it for you. + */ + experimental_reportOverride?: (params: { + key: string; + value: unknown; + entities?: EntitiesType; + }) => void | Promise; decide: (params: { key: string; entities?: EntitiesType; diff --git a/packages/vercel-flags-core/README.md b/packages/vercel-flags-core/README.md index d662d1d4..ec7ff0ab 100644 --- a/packages/vercel-flags-core/README.md +++ b/packages/vercel-flags-core/README.md @@ -24,6 +24,8 @@ const result = await client.evaluate('show-new-feature', false, { }); ``` +## Evaluation Metrics + To associate evaluation metrics with an environment, pass the `metricEnvironment` option: diff --git a/packages/vercel-flags-core/package.json b/packages/vercel-flags-core/package.json index 52468ffc..17a8e6b4 100644 --- a/packages/vercel-flags-core/package.json +++ b/packages/vercel-flags-core/package.json @@ -75,6 +75,7 @@ "dependencies": { "@vercel/functions": "^3.4.3", "@vercel/oidc": "3.5.0", + "dequal": "2.0.3", "jose": "5.2.1", "js-xxhash": "4.0.0" }, diff --git a/packages/vercel-flags-core/src/black-box.test.ts b/packages/vercel-flags-core/src/black-box.test.ts index f9ffea41..3ade1924 100644 --- a/packages/vercel-flags-core/src/black-box.test.ts +++ b/packages/vercel-flags-core/src/black-box.test.ts @@ -3742,6 +3742,320 @@ describe('Controller (black-box)', () => { }); }); + // --------------------------------------------------------------------------- + // Experiment exposure reporting + // --------------------------------------------------------------------------- + describe('experiment exposure reporting', () => { + const definitions: BundledDefinitions['definitions'] = { + flagA: { + environments: { + production: { + fallthrough: { + type: 'experiment', + }, + }, + }, + variants: ['control-a', 'treatment-a'], + variantIds: ['control-a', 'treatment-a'], + seed: 101, + experiment: { + id: 'exp_a', + base: ['user', 'key'], + weights: [0, 1], + defaultVariant: 0, + enrollmentSeed: 101, + rampId: 'ramp_a', + rampPercentage: 50, + }, + }, + flagB: { + environments: { + production: { + fallthrough: { type: 'experiment' }, + }, + }, + variants: ['control-b', 'treatment-b'], + variantIds: ['control-b', 'treatment-b'], + seed: 202, + experiment: { + id: 'exp_b', + base: ['session', 'key'], + weights: [1, 0], + defaultVariant: 0, + enrollmentSeed: 202, + }, + }, + }; + + const entity = { + user: { key: 'user_123' }, + session: { key: 'session_123' }, + }; + + it('reports one exposure with the exact evaluation entity', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: reportExposures, + }); + + const result = await client.evaluate('flagA', undefined, entity); + + expect(result).toMatchObject({ + value: 'treatment-a', + outcomeType: 'experiment', + experiment: { + id: 'exp_a', + variantId: 'treatment-a', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + assignmentReason: 'experiment', + }, + }); + expect(reportExposures).toHaveBeenCalledOnce(); + expect(reportExposures).toHaveBeenCalledWith( + [ + { + flagKey: 'flagA', + experimentId: 'exp_a', + variantId: 'treatment-a', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + assignmentReason: 'experiment', + }, + ], + entity, + ); + + await client.shutdown(); + }); + + it('does not block evaluation while reporting an exposure', async () => { + let finishReporting: () => void = () => {}; + const reporting = new Promise((resolve) => { + finishReporting = resolve; + }); + const reportExposures = vi.fn(() => reporting); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: reportExposures, + }); + + await expect( + client.evaluate('flagA', undefined, entity), + ).resolves.toMatchObject({ value: 'treatment-a' }); + expect(reportExposures).toHaveBeenCalledOnce(); + + finishReporting(); + await reporting; + await client.shutdown(); + }); + + it('reports cookie overrides without evaluating the flag', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: reportExposures, + }); + + await client.experimental_reportOverride!({ + key: 'flagA', + value: 'treatment-a', + entities: entity, + }); + + expect(reportExposures).toHaveBeenCalledOnce(); + expect(reportExposures).toHaveBeenCalledWith( + [ + { + flagKey: 'flagA', + experimentId: 'exp_a', + variantId: 'treatment-a', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + assignmentReason: 'override', + }, + ], + entity, + ); + + await client.shutdown(); + }); + + it('does not initialize override reporting without a reporter', async () => { + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + }); + + await client.experimental_reportOverride!({ + key: 'flagA', + value: true, + entities: entity, + }); + + expect(fetchMock).not.toHaveBeenCalled(); + await client.shutdown(); + }); + + it('matches object override values regardless of key order', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ + definitions: { + flagA: { + ...definitions.flagA!, + variants: [ + { enabled: true, theme: { color: 'blue', contrast: 'high' } }, + ], + variantIds: ['treatment-a'], + }, + }, + }), + experimental_reportExposures: reportExposures, + }); + + await client.experimental_reportOverride!({ + key: 'flagA', + value: { + theme: { contrast: 'high', color: 'blue' }, + enabled: true, + }, + entities: entity, + }); + + expect(reportExposures).toHaveBeenCalledWith( + [expect.objectContaining({ variantId: 'treatment-a' })], + entity, + ); + await client.shutdown(); + }); + + it('can disable exposure logging for a single evaluation', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: reportExposures, + }); + + const result = await client.evaluate('flagA', undefined, entity, { + experimental_exposureLogging: false, + }); + + expect(result.experiment?.id).toBe('exp_a'); + expect(reportExposures).not.toHaveBeenCalled(); + await client.shutdown(); + }); + + it('reports all bulk exposures in one callback', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: reportExposures, + }); + + await client.bulkEvaluate([{ key: 'flagA' }, { key: 'flagB' }], entity); + + expect(reportExposures).toHaveBeenCalledOnce(); + expect(reportExposures).toHaveBeenCalledWith( + [ + { + flagKey: 'flagA', + experimentId: 'exp_a', + variantId: 'treatment-a', + base: ['user', 'key'], + rampId: 'ramp_a', + rampPercentage: 50, + assignmentReason: 'experiment', + }, + { + flagKey: 'flagB', + experimentId: 'exp_b', + variantId: 'control-b', + base: ['session', 'key'], + assignmentReason: 'experiment', + }, + ], + entity, + ); + + await client.shutdown(); + }); + + it('can disable exposure logging for a bulk evaluation', async () => { + const reportExposures = vi.fn(); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: reportExposures, + }); + + const results = await client.bulkEvaluate( + [{ key: 'flagA' }, { key: 'flagB' }], + entity, + { experimental_exposureLogging: false }, + ); + + expect(results.flagA?.experiment?.id).toBe('exp_a'); + expect(results.flagB?.experiment?.id).toBe('exp_b'); + expect(reportExposures).not.toHaveBeenCalled(); + await client.shutdown(); + }); + + it('does not fail evaluation when the exposure reporter fails', async () => { + const error = new Error('analytics unavailable'); + const errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {}); + const client = createClient(sdkKey, { + fetch: fetchMock, + stream: false, + polling: false, + buildStep: true, + datafile: makeBundled({ definitions }), + experimental_reportExposures: () => Promise.reject(error), + }); + + const result = await client.evaluate('flagA', undefined, entity); + + expect(result.value).toBe('treatment-a'); + expect(errorSpy).toHaveBeenCalledWith( + '@vercel/flags-core: Failed to report experiment exposures', + error, + ); + await client.shutdown(); + }); + }); + // --------------------------------------------------------------------------- // Usage tracking // --------------------------------------------------------------------------- diff --git a/packages/vercel-flags-core/src/create-raw-client.ts b/packages/vercel-flags-core/src/create-raw-client.ts index bf4acb06..d2915c94 100644 --- a/packages/vercel-flags-core/src/create-raw-client.ts +++ b/packages/vercel-flags-core/src/create-raw-client.ts @@ -1,3 +1,5 @@ +import { waitUntil } from '@vercel/functions'; +import { dequal } from 'dequal/lite'; import type { bulkEvaluate, evaluate, @@ -15,7 +17,11 @@ import type { BundledDefinitions, ControllerInterface, EvaluationResult, + experimental_EvaluationOptions, + experimental_Exposure, + experimental_ReportExposures, FlagsClient, + Packed, Value, } from './types'; @@ -46,9 +52,11 @@ export function createCreateRawClient(fns: { return function createRawClient>({ controller, origin, + experimental_reportExposures, }: { controller: ControllerInterface; origin?: { provider: string; sdkKey?: string }; + experimental_reportExposures?: experimental_ReportExposures; }): FlagsClient { const id = idCount++; controllerInstanceMap.set(id, { @@ -57,6 +65,50 @@ export function createCreateRawClient(fns: { initPromise: null, }); + function report( + exposures: readonly experimental_Exposure[], + entity: Readonly, + ): void { + if (!experimental_reportExposures || exposures.length === 0) return; + + const pending = (async () => { + try { + await experimental_reportExposures(exposures, entity); + } catch (error) { + console.error( + '@vercel/flags-core: Failed to report experiment exposures', + error, + ); + } + })(); + + try { + waitUntil(pending); + } catch { + // waitUntil is best-effort; the reporter can still finish on its own. + } + } + + function getExposure( + flagKey: string, + result: EvaluationResult, + ): experimental_Exposure | null { + if (!result.experiment) return null; + return { + flagKey, + experimentId: result.experiment.id, + variantId: result.experiment.variantId, + base: result.experiment.base, + ...(result.experiment.rampId === undefined + ? {} + : { rampId: result.experiment.rampId }), + ...(result.experiment.rampPercentage === undefined + ? {} + : { rampPercentage: result.experiment.rampPercentage }), + assignmentReason: result.experiment.assignmentReason, + }; + } + const api = { origin, initialize: async () => { @@ -99,6 +151,7 @@ export function createCreateRawClient(fns: { flagKey: string, defaultValue?: T, entities?: E, + options?: experimental_EvaluationOptions, ): Promise> => { const instance = controllerInstanceMap.get(id); if (!instance?.initialized) { @@ -109,11 +162,28 @@ export function createCreateRawClient(fns: { // chain (last known value → datafile → bundled → defaultValue → throw) } } - return fns.evaluate(id, flagKey, defaultValue, entities); + const entity = entities ?? ({} as E); + const result = await fns.evaluate( + id, + flagKey, + defaultValue, + entity, + ); + if ( + experimental_reportExposures && + options?.experimental_exposureLogging !== false + ) { + const exposure = getExposure(flagKey, result); + if (exposure) { + report([exposure], entity as unknown as Readonly); + } + } + return result; }, bulkEvaluate: async ( flags: BulkEvaluateInput[], entities?: E, + options?: experimental_EvaluationOptions, ): Promise>> => { const instance = controllerInstanceMap.get(id); if (!instance?.initialized) { @@ -124,7 +194,73 @@ export function createCreateRawClient(fns: { // chain (last known value → datafile → bundled → defaultValue → throw) } } - return fns.bulkEvaluate(id, flags, entities); + const entity = entities ?? ({} as E); + const results = await fns.bulkEvaluate(id, flags, entity); + if ( + experimental_reportExposures && + options?.experimental_exposureLogging !== false + ) { + const exposures: experimental_Exposure[] = []; + const seen = new Set(); + for (const flag of flags) { + if (seen.has(flag.key)) continue; + seen.add(flag.key); + const result = results[flag.key]; + if (!result) continue; + const exposure = getExposure(flag.key, result); + if (exposure) exposures.push(exposure); + } + report(exposures, entity as unknown as Readonly); + } + return results; + }, + experimental_reportOverride: async ({ + key, + value, + entities, + }: { + key: string; + value: T; + entities?: E; + }): Promise => { + if (!experimental_reportExposures) return; + + try { + const instance = controllerInstanceMap.get(id); + if (!instance?.initialized) await api.initialize(); + const datafile = await fns.getDatafile(id); + const definition = datafile.definitions[key] as Packed.FlagDefinition; + const experiment = definition?.experiment; + if (!experiment) return; + + const variantIndex = definition.variants.findIndex((variant) => + dequal(variant, value), + ); + const variantId = + variantIndex < 0 + ? null + : (definition.variantIds?.[variantIndex] ?? null); + const entity = entities ?? ({} as E); + report( + [ + { + flagKey: key, + experimentId: experiment.id, + variantId, + base: experiment.base, + rampId: experiment.rampId, + rampPercentage: experiment.rampPercentage, + assignmentReason: 'override', + }, + ], + entity as unknown as Readonly, + ); + } catch (error) { + console.error( + '@vercel/flags-core: Failed to report experiment override', + error, + ); + } }, }; return api; diff --git a/packages/vercel-flags-core/src/evaluate.test.ts b/packages/vercel-flags-core/src/evaluate.test.ts index 6ecb9312..dc00b1e9 100644 --- a/packages/vercel-flags-core/src/evaluate.test.ts +++ b/packages/vercel-flags-core/src/evaluate.test.ts @@ -2700,6 +2700,216 @@ describe('evaluate', () => { }); }); +describe('experiment metadata', () => { + const definition = { + environments: { + production: { + rules: [ + { + conditions: [[['user', 'country'], Comparator.EQ, 'DE']], + outcome: { type: 'experiment' }, + }, + ], + fallthrough: 0, + }, + }, + variants: ['control', 'treatment'], + variantIds: ['flag-control', 'flag-treatment'], + seed: 123, + experiment: { + id: 'exp_checkout', + base: ['user', 'key'], + weights: [0, 1], + defaultVariant: 0, + enrollmentSeed: 456, + rampId: 'ramp_1', + rampPercentage: 100, + }, + } satisfies Packed.FlagDefinition; + + it('randomizes an enrolled experiment outcome', () => { + expect( + evaluate({ + definition, + environment: 'production', + entities: { user: { key: 'user_123', country: 'DE' } }, + }), + ).toEqual({ + value: 'treatment', + variantId: 'flag-treatment', + reason: ResolutionReason.RULE_MATCH, + outcomeType: OutcomeType.EXPERIMENT, + experiment: { + id: 'exp_checkout', + variantId: 'flag-treatment', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 100, + assignmentReason: 'experiment', + }, + }); + }); + + it('marks a missing experiment base as not enrolled', () => { + expect( + evaluate({ + definition, + environment: 'production', + entities: { user: { country: 'DE' } }, + }), + ).toEqual({ + value: 'control', + variantId: 'flag-control', + reason: ResolutionReason.RULE_MATCH, + outcomeType: OutcomeType.EXPERIMENT, + experiment: { + id: 'exp_checkout', + variantId: 'flag-control', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 100, + assignmentReason: 'not-enrolled', + }, + }); + }); + + it('marks a fixed outcome as a non-randomized variant exposure', () => { + expect( + evaluate({ + definition, + environment: 'production', + entities: { user: { key: 'user_123', country: 'US' } }, + }), + ).toEqual({ + value: 'control', + variantId: 'flag-control', + reason: ResolutionReason.FALLTHROUGH, + outcomeType: OutcomeType.VALUE, + experiment: { + id: 'exp_checkout', + variantId: 'flag-control', + base: ['user', 'key'], + rampId: 'ramp_1', + rampPercentage: 100, + assignmentReason: 'variant', + }, + }); + }); + + it('marks an ordinary split as a non-experiment split exposure', () => { + expect( + evaluate({ + definition: { + ...definition, + environments: { + production: { + fallthrough: { + type: 'split', + base: ['user', 'key'], + weights: [1, 0], + defaultVariant: 0, + }, + }, + }, + }, + environment: 'production', + entities: { user: { key: 'user_123' } }, + }), + ).toMatchObject({ + value: 'control', + outcomeType: OutcomeType.SPLIT, + experiment: { assignmentReason: 'split' }, + }); + }); + + it('marks direct targets as targeted exposures', () => { + expect( + evaluate({ + definition: { + ...definition, + environments: { + production: { + targets: [{ user: { key: ['user_123'] } }], + fallthrough: { type: 'experiment' }, + }, + }, + }, + environment: 'production', + entities: { user: { key: 'user_123' } }, + }), + ).toMatchObject({ + value: 'control', + experiment: { assignmentReason: 'targeted' }, + }); + }); + + it('preserves enrolled assignments as ramp percentage increases', () => { + const makeExperimentDefinition = ( + rampPercentage: number, + ): Packed.FlagDefinition => ({ + ...definition, + environments: { + production: { fallthrough: { type: 'experiment' } }, + }, + experiment: { + ...definition.experiment, + weights: [1, 1], + rampPercentage, + }, + }); + const splitDefinition: Packed.FlagDefinition = { + ...definition, + environments: { + production: { + fallthrough: { + type: 'split', + base: definition.experiment.base, + weights: [1, 1], + defaultVariant: 0, + }, + }, + }, + experiment: undefined, + }; + let enrolledAtTwenty = 0; + let newlyEnrolled = 0; + + for (let index = 0; index < 500; index++) { + const entities = { user: { key: `user_${index}` } }; + const atTwenty = evaluate({ + definition: makeExperimentDefinition(20), + environment: 'production', + entities, + }); + const atEighty = evaluate({ + definition: makeExperimentDefinition(80), + environment: 'production', + entities, + }); + + if (atTwenty.experiment?.assignmentReason === 'experiment') { + enrolledAtTwenty++; + expect(atEighty.experiment?.assignmentReason).toBe('experiment'); + expect(atEighty.variantId).toBe(atTwenty.variantId); + } else if (atEighty.experiment?.assignmentReason === 'experiment') { + newlyEnrolled++; + } + + if (atEighty.experiment?.assignmentReason === 'experiment') { + const withoutExperiment = evaluate({ + definition: splitDefinition, + environment: 'production', + entities, + }); + expect(atEighty.variantId).toBe(withoutExperiment.variantId); + } + } + + expect(enrolledAtTwenty).toBeGreaterThan(0); + expect(newlyEnrolled).toBeGreaterThan(0); + }); +}); + describe('bulkEvaluate', () => { it('evaluates multiple flags against shared entities, segments, and environment', () => { const activeDef: Packed.FlagDefinition = { diff --git a/packages/vercel-flags-core/src/evaluate.ts b/packages/vercel-flags-core/src/evaluate.ts index 1e51f82c..22c4a9df 100644 --- a/packages/vercel-flags-core/src/evaluate.ts +++ b/packages/vercel-flags-core/src/evaluate.ts @@ -3,6 +3,8 @@ import { Comparator, type EvaluationParams, type EvaluationResult, + type experimental_ExperimentAssignment, + type experimental_ExperimentAssignmentReason, OutcomeType, Packed, ResolutionReason, @@ -40,14 +42,14 @@ function boundaryFor(numerator: number, denominator: number): number { // symbol-keyed props) and serialize cleanly across the RSC boundary; entries // are GC'd with the datafile. Split boundaries are static per outcome, so the // cumulative cut points are computed once and reused across evaluations. -const splitBoundariesCache = new WeakMap(); +const splitBoundariesCache = new WeakMap(); const compiledRegexCache = new WeakMap(); /** * Cumulative hash boundaries for a split, one per variant in index order. * Variant `i` is served for hashes in `[boundaries[i-1], boundaries[i])`. */ -function getSplitBoundaries(outcome: Packed.SplitOutcome): number[] { +function getSplitBoundaries(outcome: { weights: number[] }): number[] { const cached = splitBoundariesCache.get(outcome); if (cached) return cached; const total = sum(outcome.weights); @@ -414,13 +416,73 @@ function getVariant( }; } -function handleOutcome( +type WeightedAssignment = { + base: Packed.EntityAccessor; + weights: number[]; + defaultVariant: Packed.VariantIndex; +}; + +function getWeightedVariantIndex( + params: EvaluationParams, + assignment: WeightedAssignment, + seed: number | undefined, +): Packed.VariantIndex { + const lhs = access(assignment.base, params); + + if (typeof lhs !== 'string') return assignment.defaultVariant; + + const bucket = hashInput(lhs, seed); + const boundaries = getSplitBoundaries(assignment); + for (let index = 0; index < boundaries.length; index++) { + if (bucket < (boundaries[index] as number)) return index; + } + + // Only reached when the weights sum to 0 (every boundary is NaN). + return assignment.defaultVariant; +} + +function experimentAssignment( + experiment: Packed.experimental_ExperimentDefinition, + variantId: VariantId | null, + assignmentReason: experimental_ExperimentAssignmentReason, +): experimental_ExperimentAssignment | undefined { + if (variantId === null) return undefined; + return { + id: experiment.id, + variantId, + base: experiment.base, + rampId: experiment.rampId, + rampPercentage: experiment.rampPercentage, + assignmentReason, + }; +} + +function outcomeAssignmentReason( + outcome: Packed.Outcome, +): experimental_ExperimentAssignmentReason { + if (typeof outcome === 'number') return 'variant'; + switch (outcome.type) { + case 'experiment': + return 'experiment'; + case 'split': + return 'split'; + case 'rollout': + return 'rollout'; + default: { + const { type } = outcome; + return exhaustivenessCheck(type); + } + } +} + +function resolveOutcome( params: EvaluationParams, outcome: Packed.Outcome, ): { value: T; outcomeType: OutcomeType; variantId: VariantId | null; + experiment?: experimental_ExperimentAssignment; } { if (typeof outcome === 'number') { const variant = getVariant(params.definition, outcome); @@ -431,38 +493,58 @@ function handleOutcome( } switch (outcome.type) { case 'split': { - const lhs = access(outcome.base, params); - const defaultOutcome = getVariant( - params.definition, - outcome.defaultVariant, + const index = getWeightedVariantIndex( + params, + outcome, + params.definition.seed, ); - - // serve the default variant if the lhs is not a string - if (typeof lhs !== 'string') { - return { - ...defaultOutcome, - outcomeType: OutcomeType.SPLIT, - }; + return { + ...getVariant(params.definition, index), + outcomeType: OutcomeType.SPLIT, + }; + } + case 'experiment': { + const experiment = params.definition.experiment; + if (!experiment) { + throw new Error('@vercel/flags-core: Experiment not found'); } - const bucket = hashInput(lhs, params.definition.seed); - const boundaries = getSplitBoundaries(outcome); - - // Return the first variant whose cumulative boundary covers the bucket. - for (let index = 0; index < boundaries.length; index++) { - if (bucket < (boundaries[index] as number)) { - return { - ...getVariant(params.definition, index), - outcomeType: OutcomeType.SPLIT, - }; - } + const unitValue = access(experiment.base, params); + const defaultVariant = getVariant( + params.definition, + experiment.defaultVariant, + ); + const assignment = ( + variant: typeof defaultVariant, + assignmentReason: experimental_ExperimentAssignmentReason, + ) => ({ + ...variant, + outcomeType: OutcomeType.EXPERIMENT, + experiment: experimentAssignment( + experiment, + variant.variantId, + assignmentReason, + ), + }); + + if (typeof unitValue !== 'string') { + return assignment(defaultVariant, 'not-enrolled'); } - // Only reached when the weights sum to 0 (every boundary is NaN). - return { - ...defaultOutcome, - outcomeType: OutcomeType.SPLIT, - }; + const rampPercentage = experiment.rampPercentage ?? 100; + const enrolled = + rampPercentage >= 100 || + (rampPercentage > 0 && + hashInput(unitValue, experiment.enrollmentSeed) < + boundaryFor(rampPercentage, 100)); + if (!enrolled) return assignment(defaultVariant, 'not-enrolled'); + + const index = getWeightedVariantIndex( + params, + experiment, + params.definition.seed, + ); + return assignment(getVariant(params.definition, index), 'experiment'); } case 'rollout': { const lhs = access(outcome.base, params); @@ -557,6 +639,30 @@ function handleOutcome( } } +function handleOutcome( + params: EvaluationParams, + outcome: Packed.Outcome, + assignmentReason?: experimental_ExperimentAssignmentReason, +): { + value: T; + outcomeType: OutcomeType; + variantId: VariantId | null; + experiment?: experimental_ExperimentAssignment; +} { + const result = resolveOutcome(params, outcome); + const experiment = params.definition.experiment; + if (!experiment || result.experiment) return result; + + return { + ...result, + experiment: experimentAssignment( + experiment, + result.variantId, + assignmentReason ?? outcomeAssignmentReason(outcome), + ), + }; +} + /** * Evaluates a single feature flag. * @@ -623,7 +729,7 @@ export function evaluate( ); if (matchedIndex > -1) { - return Object.assign(handleOutcome(params, matchedIndex), { + return Object.assign(handleOutcome(params, matchedIndex, 'targeted'), { reason: ResolutionReason.TARGET_MATCH as const, }) satisfies EvaluationResult; } diff --git a/packages/vercel-flags-core/src/index.common.ts b/packages/vercel-flags-core/src/index.common.ts index a8834192..1364b89b 100644 --- a/packages/vercel-flags-core/src/index.common.ts +++ b/packages/vercel-flags-core/src/index.common.ts @@ -18,6 +18,10 @@ export { type DatafileInput, type EvaluationParams, type EvaluationResult, + type experimental_EvaluationOptions, + type experimental_ExperimentAssignment, + type experimental_Exposure, + type experimental_ReportExposures, type FlagsClient, type Packed, type PollingOptions, diff --git a/packages/vercel-flags-core/src/index.make.test.ts b/packages/vercel-flags-core/src/index.make.test.ts index fa139e28..429be61f 100644 --- a/packages/vercel-flags-core/src/index.make.test.ts +++ b/packages/vercel-flags-core/src/index.make.test.ts @@ -120,6 +120,27 @@ describe('make', () => { expect(client).toBeDefined(); }); + it('should pass experimental_reportExposures to the raw client, not the controller', () => { + const createRawClient = createMockCreateRawClient(); + const { createClient } = make(createRawClient); + const reportExposures = vi.fn(); + + createClient('vf_server_test_key', { + stream: false, + experimental_reportExposures: reportExposures, + }); + + expect(Controller).toHaveBeenCalledWith({ + auth: expect.objectContaining({ sdkKey: 'vf_server_test_key' }), + stream: false, + }); + expect(createRawClient).toHaveBeenCalledWith({ + controller: expect.any(Object), + origin: { provider: 'vercel', sdkKey: 'vf_server_test_key' }, + experimental_reportExposures: reportExposures, + }); + }); + it('should throw for empty SDK key', () => { const createRawClient = createMockCreateRawClient(); const { createClient } = make(createRawClient); diff --git a/packages/vercel-flags-core/src/index.make.ts b/packages/vercel-flags-core/src/index.make.ts index 19343c94..9e83ea1c 100644 --- a/packages/vercel-flags-core/src/index.make.ts +++ b/packages/vercel-flags-core/src/index.make.ts @@ -5,20 +5,31 @@ import { Controller, type ControllerOptions } from './controller'; import { Authentication } from './controller/auth'; import type { createCreateRawClient } from './create-raw-client'; -import type { FlagsClient } from './types'; +import type { experimental_ReportExposures, FlagsClient } from './types'; /** * Options for createClient */ -export type CreateClientOptions = Omit; +export type CreateClientOptions> = Omit< + ControllerOptions, + 'auth' +> & { + /** + * Reports experiment exposures produced by evaluation calls. + * + * @remarks This API is not supported for general use yet. Do not use it + * unless Vercel has explicitly enabled it for you. + */ + experimental_reportExposures?: experimental_ReportExposures; +}; type CreateClient = { >( - options: CreateClientOptions, + options: CreateClientOptions, ): FlagsClient; >( sdkKeyOrConnectionString?: string, - options?: CreateClientOptions, + options?: CreateClientOptions, ): FlagsClient; }; @@ -35,15 +46,15 @@ export function make( // - data source must specify the environment & projectId as sdkKey has that info // - "reuse" functionality relies on the data source having the data for all envs function createClient>( - options: CreateClientOptions, + options: CreateClientOptions, ): FlagsClient; function createClient>( sdkKeyOrConnectionString?: string, - options?: CreateClientOptions, + options?: CreateClientOptions, ): FlagsClient; function createClient>( - sdkKeyOrConnectionStringOrOptions?: string | CreateClientOptions, - options?: CreateClientOptions, + sdkKeyOrConnectionStringOrOptions?: string | CreateClientOptions, + options?: CreateClientOptions, ): FlagsClient { const optionsOnly = typeof sdkKeyOrConnectionStringOrOptions === 'object' && @@ -55,13 +66,16 @@ export function make( ? sdkKeyOrConnectionStringOrOptions : options; + const { experimental_reportExposures, ...controllerOptions } = + createClientOptions ?? {}; const auth = new Authentication(sdkKeyOrConnectionString); // sdk key contains the environment - const controller = new Controller({ auth, ...createClientOptions }); + const controller = new Controller({ auth, ...controllerOptions }); return createRawClient({ controller, origin: { provider: 'vercel', sdkKey: auth.sdkKey }, + ...(experimental_reportExposures ? { experimental_reportExposures } : {}), }); } diff --git a/packages/vercel-flags-core/src/types.ts b/packages/vercel-flags-core/src/types.ts index f12f3382..42b4ac8d 100644 --- a/packages/vercel-flags-core/src/types.ts +++ b/packages/vercel-flags-core/src/types.ts @@ -125,6 +125,64 @@ export type BulkEvaluateInput = { defaultValue?: T; }; +/** Options that control side effects of an evaluation call. */ +export type experimental_EvaluationOptions = { + /** + * Whether experiment exposures should be reported for this evaluation. + * @default true + */ + experimental_exposureLogging?: boolean; +}; + +export type experimental_ExperimentAssignmentReason = + | 'experiment' + | 'not-enrolled' + | 'targeted' + | 'split' + | 'variant' + | 'rollout' + | 'override'; + +/** Information about the experiment linked to an evaluated flag value. */ +export type experimental_ExperimentAssignment = { + /** Experiment identifier. */ + id: string; + /** Identifier of the selected experiment variant. */ + variantId: string; + /** Entity path on which the experiment assignment is based. */ + base: Packed.EntityAccessor; + /** Identifier of the ramp active for this assignment. */ + rampId?: string; + /** Percentage of eligible units included in the ramp, from 0 through 100. */ + rampPercentage?: number; + /** How this evaluation received its value. */ + assignmentReason: experimental_ExperimentAssignmentReason; +}; + +/** An experiment exposure passed to a client's exposure reporter. */ +export type experimental_Exposure = { + /** Flag whose evaluation produced the exposure. */ + flagKey: FlagKey; + /** Experiment identifier. */ + experimentId: string; + /** Identifier of the selected experiment variant. */ + variantId: string | null; + /** Entity path on which the experiment assignment is based. */ + base: Packed.EntityAccessor; + /** Identifier of the ramp active for this assignment. */ + rampId?: string; + /** Percentage of eligible units included in the ramp, from 0 through 100. */ + rampPercentage?: number; + /** How this evaluation received its value. */ + assignmentReason: experimental_ExperimentAssignmentReason; +}; + +/** Reports experiment exposures produced by one evaluation call. */ +export type experimental_ReportExposures> = ( + exposures: readonly experimental_Exposure[], + entity: Readonly, +) => void | Promise; + /** * A client for Vercel Flags */ @@ -145,12 +203,14 @@ export type FlagsClient> = { * @param flagKey * @param defaultValue * @param entities + * @param options Evaluation side-effect options. * @returns */ evaluate: ( flagKey: string, defaultValue?: T, entities?: E, + options?: experimental_EvaluationOptions, ) => Promise>; /** * Evaluate multiple feature flags against the same entities in a single call. @@ -162,12 +222,25 @@ export type FlagsClient> = { * * @param flags Array of `{ key, defaultValue? }` entries to evaluate. * @param entities Shared entities used for every flag in the bulk call. + * @param options Evaluation side-effect options. * @returns Object mapping each key to its EvaluationResult. */ bulkEvaluate: ( flags: BulkEvaluateInput[], entities?: E, + options?: experimental_EvaluationOptions, ) => Promise>>; + /** + * Report a Flags SDK override without evaluating the provider value. + * + * @remarks This API is not supported for general use yet. Do not use it + * unless Vercel has explicitly enabled it for you. + */ + experimental_reportOverride?: (params: { + key: string; + value: T; + entities?: E; + }) => Promise; /** * Retrieve the latest datafile during startup, and set up subscriptions if needed. */ @@ -252,6 +325,8 @@ export type EvaluationResult = * The variant we want to report for o11y */ variantId: VariantId | null; + /** Experiment metadata when the flag is linked to an experiment. */ + experiment?: experimental_ExperimentAssignment; /** * Indicates why the flag evaluated to a certain value */ @@ -266,6 +341,7 @@ export type EvaluationResult = errorMessage: string; errorCode?: ErrorCode; outcomeType?: never; + experiment?: never; /** * The variant we want to report for o11y */ @@ -309,6 +385,8 @@ export enum OutcomeType { SPLIT = 'split', /** When the outcome type was a progressive rollout */ ROLLOUT = 'rollout', + /** When the experiment assignment mechanism produced the value */ + EXPERIMENT = 'experiment', } /** @@ -542,8 +620,26 @@ export namespace Original { * Once all slots are exhausted, the rollout is complete (100% rollToVariant). */ slots: { promille: number; durationMs: number }[]; + } + | { + type: 'experiment'; }; + export type experimental_ExperimentDefinition = { + id: string; + /** Entity attribute used as the experiment unit. */ + base: EntityAccessor; + /** Distribution keyed by flag variant ID. */ + weights: Record; + /** Flag variant used when the base attribute does not exist. */ + defaultVariantId: VariantId; + /** Stable seed used only for experiment enrollment. */ + enrollmentSeed: number; + rampId?: string; + /** Percentage from 0 through 100. */ + rampPercentage?: number; + }; + export type SegmentAllOutcome = { type: 'all'; }; @@ -671,6 +767,8 @@ export namespace Original { export type FlagDefinition = { variants: FlagVariant[]; + /** Experiment linked to this flag. */ + experiment?: experimental_ExperimentDefinition; environments: Record; /** @@ -766,6 +864,23 @@ export namespace Packed { slots: [number, number][]; }; + export type experimental_ExperimentDefinition = { + /** Experiment identifier. */ + id: string; + /** Entity path used as the experiment unit. */ + base: EntityAccessor; + /** Distribution indexed by the corresponding flag variant. */ + weights: number[]; + /** Flag variant used when the experiment base is unavailable. */ + defaultVariant: VariantIndex; + /** Stable seed used only for experiment enrollment. */ + enrollmentSeed: number; + /** Identifier of the ramp active for this experiment. */ + rampId?: string; + /** Percentage of eligible units included in the ramp, from 0 through 100. */ + rampPercentage?: number; + }; + export type SegmentAllOutcome = 1; export type SegmentSplitOutcome = { @@ -784,7 +899,13 @@ export namespace Packed { export type SegmentOutcome = SegmentAllOutcome | SegmentSplitOutcome; - export type Outcome = VariantIndex | SplitOutcome | RolloutOutcome; + export type experimental_ExperimentOutcome = { type: 'experiment' }; + + export type Outcome = + | VariantIndex + | SplitOutcome + | RolloutOutcome + | experimental_ExperimentOutcome; // an array means it's an entity, the string "segment" means a segment export type EntityAccessor = (string | number)[]; @@ -893,6 +1014,8 @@ export namespace Packed { variantIds?: string[]; /** variants, packed down to just their values */ variants: Value[]; + /** Experiment linked to this flag. */ + experiment?: experimental_ExperimentDefinition; /** environments */ environments: Record; /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 20dd45f6..f2661203 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -908,6 +908,9 @@ importers: '@vercel/oidc': specifier: 3.5.0 version: 3.5.0 + dequal: + specifier: 2.0.3 + version: 2.0.3 jose: specifier: 5.2.1 version: 5.2.1